    001/*
    002 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    003 *
    004 * Project and contact information: http://www.cascading.org/
    005 *
    006 * This file is part of the Cascading project.
    007 *
    008 * Licensed under the Apache License, Version 2.0 (the "License");
    009 * you may not use this file except in compliance with the License.
    010 * You may obtain a copy of the License at
    011 *
    012 *     http://www.apache.org/licenses/LICENSE-2.0
    013 *
    014 * Unless required by applicable law or agreed to in writing, software
    015 * distributed under the License is distributed on an "AS IS" BASIS,
    016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    017 * See the License for the specific language governing permissions and
    018 * limitations under the License.
    019 */
    020
    021package cascading.tuple.collect;
    022
    023import java.util.ArrayList;
    024import java.util.List;
    025import java.util.Properties;
    026
    027import cascading.property.Props;
    028import cascading.util.Util;
    029
    030/**
    031 * Class SpillableProps is a fluent interface for building properties to be passed to a
    032 * {@link cascading.flow.FlowConnector} before creating new {@link cascading.flow.Flow} instances.
    033 *
    034 * @see SpillableTupleList
    035 * @see SpillableTupleMap
    036 */
    037public class SpillableProps extends Props
    038  {
    039  /**
    040   * Whether to enable compression of the spills or not, on by default.
    041   *
    042   * @see Boolean#parseBoolean(String)
    043   */
    044  public static final String SPILL_COMPRESS = "cascading.spill.compress";
    045
    046  /** A comma delimited list of possible codecs to try. This is platform dependent. */
    047  public static final String SPILL_CODECS = "cascading.spill.codecs";
    048
    049  /** Number of tuples to hold in memory before spilling them to disk. */
    050  public static final String LIST_THRESHOLD = "cascading.spill.list.threshold";
    051
    052  /** The total number of tuple values (not keys) to attempt to keep in memory. */
    053  public static final String MAP_THRESHOLD = "cascading.spill.map.threshold";
    054
    055  /**
    056   * The initial hash map capacity.
    057   *
    058   * @see java.util.HashMap
    059   */
    060  public static final String MAP_CAPACITY = "cascading.spill.map.capacity";
    061
    062  /**
    063   * The initial hash map load factor.
    064   *
    065   * @see java.util.HashMap
    066   */
    067  public static final String MAP_LOADFACTOR = "cascading.spill.map.loadfactor";
    068
    069  public static final int defaultListThreshold = 10 * 1000;
    070
    071  public static final int defaultMapThreshold = 10 * 1000;
    072  public static final int defaultMapInitialCapacity = 100 * 1000;
    073  public static final float defaultMapLoadFactor = 0.75f;
    074
    075  boolean compressSpill = true;
    076  List<String> codecs = new ArrayList<String>();
    077
    078  int listSpillThreshold = defaultListThreshold;
    079
    080  int mapSpillThreshold = defaultMapThreshold;
    081  int mapInitialCapacity = defaultMapInitialCapacity;
    082  float mapLoadFactor = defaultMapLoadFactor;
    083
    084  /**
    085   * Creates a new SpillableProps instance.
    086   *
    087   * @return SpillableProps instance
    088   */
    089  public static SpillableProps spillableProps()
    090    {
    091    return new SpillableProps();
    092    }
    093
    094  public SpillableProps()
    095    {
    096    }
    097
    098  public boolean isCompressSpill()
    099    {
    100    return compressSpill;
    101    }
    102
    103  /**
    104   * Method setCompressSpill either enables or disables spill compression. Enabled by default.
    105   * <p/>
    106   * Spill compression relies on properly configured and available codecs. See {@link #setCodecs(java.util.List)}.
    107   *
    108   * @param compressSpill type boolean
    109   * @return this
    110   */
    111  public SpillableProps setCompressSpill( boolean compressSpill )
    112    {
    113    this.compressSpill = compressSpill;
    114
    115    return this;
    116    }
    117
    118  public List<String> getCodecs()
    119    {
    120    return codecs;
    121    }
    122
    123  /**
    124   * Method setCodecs sets list of possible codec class names to use. They will be loaded in order, if available.
    125   * <p/>
    126   * This is platform dependent.
    127   *
    128   * @param codecs type list
    129   * @return this
    130   */
    131  public SpillableProps setCodecs( List<String> codecs )
    132    {
    133    this.codecs = codecs;
    134
    135    return this;
    136    }
    137
    138  /**
    139   * Method addCodecs adds a list of possible codec class names to use. They will be loaded in order, if available.
    140   * <p/>
    141   * This is platform dependent.
    142   *
    143   * @param codecs type list
    144   */
    145  public SpillableProps addCodecs( List<String> codecs )
    146    {
    147    this.codecs.addAll( codecs );
    148
    149    return this;
    150    }
    151
    152  /**
    153   * Method addCodec adds a codec class names to use.
    154   * <p/>
    155   * This is platform dependent.
    156   *
    157   * @param codec type String
    158   */
    159  public SpillableProps addCodec( String codec )
    160    {
    161    this.codecs.add( codec );
    162
    163    return this;
    164    }
    165
    166  public int getListSpillThreshold()
    167    {
    168    return listSpillThreshold;
    169    }
    170
    171  /**
    172   * Method setListSpillThreshold sets the number of tuples to hold in memory before spilling them to disk.
    173   *
    174   * @param listSpillThreshold of type int
    175   * @return this
    176   */
    177  public SpillableProps setListSpillThreshold( int listSpillThreshold )
    178    {
    179    this.listSpillThreshold = listSpillThreshold;
    180
    181    return this;
    182    }
    183
    184  public int getMapSpillThreshold()
    185    {
    186    return mapSpillThreshold;
    187    }
    188
    189  /**
    190   * Method setMapSpillThreshold the total number of tuple values (not keys) to attempt to keep in memory.
    191   * <p/>
    192   * The default implementation cannot spill Map keys to disk.
    193   *
    194   * @param mapSpillThreshold of type int
    195   * @return this
    196   */
    197  public SpillableProps setMapSpillThreshold( int mapSpillThreshold )
    198    {
    199    this.mapSpillThreshold = mapSpillThreshold;
    200
    201    return this;
    202    }
    203
    204  public int getMapInitialCapacity()
    205    {
    206    return mapInitialCapacity;
    207    }
    208
    209  /**
    210   * Method setMapInitialCapacity sets the default capacity to be used by the backing Map implementation.
    211   *
    212   * @param mapInitialCapacity type int
    213   * @return this
    214   */
    215  public SpillableProps setMapInitialCapacity( int mapInitialCapacity )
    216    {
    217    this.mapInitialCapacity = mapInitialCapacity;
    218
    219    return this;
    220    }
    221
    222  public float getMapLoadFactor()
    223    {
    224    return mapLoadFactor;
    225    }
    226
    227  /**
    228   * Method setMapLoadFactor sets the default load factor to be used by the backing Map implementation.
    229   *
    230   * @param mapLoadFactor type float
    231   * @return this
    232   */
    233  public SpillableProps setMapLoadFactor( float mapLoadFactor )
    234    {
    235    this.mapLoadFactor = mapLoadFactor;
    236
    237    return this;
    238    }
    239
    240  @Override
    241  protected void addPropertiesTo( Properties properties )
    242    {
    243    for( String codec : codecs )
    244      {
    245      String codecs = (String) properties.get( SPILL_CODECS );
    246
    247      properties.put( SPILL_CODECS, Util.join( ",", Util.removeNulls( codecs, codec ) ) );
    248      }
    249
    250    properties.setProperty( SPILL_COMPRESS, Boolean.toString( compressSpill ) );
    251    properties.setProperty( LIST_THRESHOLD, Integer.toString( listSpillThreshold ) );
    252
    253    properties.setProperty( MAP_THRESHOLD, Integer.toString( mapSpillThreshold ) );
    254    properties.setProperty( MAP_CAPACITY, Integer.toString( mapInitialCapacity ) );
    255    properties.setProperty( MAP_LOADFACTOR, Float.toString( mapLoadFactor ) );
    256    }
    257  }