    001/*
    002 * Copyright (c) 2016 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.flow.planner;
    023
    024import java.io.Serializable;
    025import java.util.HashSet;
    026import java.util.LinkedList;
    027import java.util.Map;
    028import java.util.Set;
    029
    030import cascading.tuple.Fields;
    031
    032import static cascading.tuple.Fields.asDeclaration;
    033
    034/** Class Scope is an internal representation of the linkages between operations. */
    035public class Scope implements Serializable
    036  {
    037  /** Enum Kind */
    038  public enum Kind
    039    {
    040      TAP, EACH, EVERY, GROUPBY, COGROUP, HASHJOIN, MERGE
    041    }
    042
    043  /** Field name */
    044  private String name;
    045  private LinkedList<String> priorNames = new LinkedList<>();
    046
    047  /** Field ordinal */
    048  private Integer ordinal = 0; // do not copy
    049  /** Field isStreamed is not accumulated. currently only relevant to HashJoin * */
    050  private boolean isNonBlocking = true; // do not copy
    051
    052  // copied
    053
    054  /** Field kind */
    055  private Kind kind;
    056  /** Field incomingPassThroughFields */
    057  private Fields incomingPassThroughFields;
    058  /** Field remainderPassThroughFields */
    059  private Fields remainderPassThroughFields;
    060
    061  /** Field argumentSelector */
    062  private Fields operationArgumentFields;
    063  /** Field declaredFields */
    064  private Fields operationDeclaredFields; // fields declared by the operation
    065
    066  /** Field groupingSelectors */
    067  private Map<String, Fields> keySelectors;
    068  /** Field sortingSelectors */
    069  private Map<String, Fields> sortingSelectors;
    070
    071  /** Field outGroupingSelector */
    072  private Fields outGroupingSelector;
    073  /** Field outGroupingFields */
    074  private Fields outGroupingFields; // all key fields
    075
    076  /** Field outValuesSelector */
    077  private Fields outValuesSelector;
    078  /** Field outValuesFields */
    079  private Fields outValuesFields; // all value fields, includes keys
    080
    081  /** Default constructor. */
    082  public Scope()
    083    {
    084    }
    085
    086  /**
    087   * Copy constructor
    088   *
    089   * @param scope of type Scope
    090   */
    091  public Scope( Scope scope )
    092    {
    093    this.name = scope.getName();
    094    copyFields( scope );
    095    }
    096
    097  /**
    098   * Tap constructor
    099   *
    100   * @param outFields of type Fields
    101   */
    102  public Scope( Fields outFields )
    103    {
    104    this.kind = Kind.TAP;
    105
    106    if( outFields == null )
    107      throw new IllegalArgumentException( "fields may not be null" );
    108
    109    this.outGroupingFields = outFields;
    110    this.outValuesFields = outFields;
    111    }
    112
    113  /**
    114   * Constructor Scope creates a new Scope instance. Used by classes Each and Every.
    115   *
    116   * @param name                      of type String
    117   * @param kind                      of type Kind
    118   * @param incomingPassThroughFields //   * @param remainderPassThroughFields   of type Fields
    119   * @param operationArgumentFields   of type Fields
    120   * @param operationDeclaredFields   of type Fields
    121   * @param outGroupingFields         of type Fields
    122   * @param outValuesFields           of type Fields
    123   */
    124  public Scope( String name, Kind kind, Fields incomingPassThroughFields, Fields remainderPassThroughFields, Fields operationArgumentFields, Fields operationDeclaredFields, Fields outGroupingFields, Fields outValuesFields )
    125    {
    126    this.name = name;
    127    this.kind = kind;
    128    this.incomingPassThroughFields = incomingPassThroughFields;
    129    this.remainderPassThroughFields = remainderPassThroughFields;
    130    this.operationArgumentFields = operationArgumentFields;
    131    this.operationDeclaredFields = operationDeclaredFields;
    132
    133    if( outGroupingFields == null )
    134      throw new IllegalArgumentException( "grouping may not be null" );
    135
    136    if( outValuesFields == null )
    137      throw new IllegalArgumentException( "values may not be null" );
    138
    139    if( kind == Kind.EACH || kind == Kind.EVERY )
    140      {
    141      this.outGroupingSelector = outGroupingFields;
    142      this.outGroupingFields = asDeclaration( outGroupingFields );
    143      this.outValuesSelector = outValuesFields;
    144      this.outValuesFields = asDeclaration( outValuesFields );
    145      }
    146    else
    147      {
    148      throw new IllegalArgumentException( "may not use the constructor for kind: " + kind );
    149      }
    150    }
    151
    152  /**
    153   * Constructor Scope creates a new Scope instance. Used by the Group class.
    154   *
    155   * @param name                    of type String
    156   * @param operationDeclaredFields of type Fields
    157   * @param outGroupingFields       of type Fields
    158   * @param keySelectors            of type Map<String, Fields>
    159   * @param sortingSelectors        of type Fields
    160   * @param outValuesFields         of type Fields
    161   * @param kind                    of type boolean
    162   */
    163  public Scope( String name, Fields operationDeclaredFields, Fields outGroupingFields, Map<String, Fields> keySelectors, Map<String, Fields> sortingSelectors, Fields outValuesFields, Kind kind )
    164    {
    165    this.name = name;
    166    this.kind = kind;
    167
    168    if( keySelectors == null )
    169      throw new IllegalArgumentException( "grouping may not be null" );
    170
    171    if( outValuesFields == null )
    172      throw new IllegalArgumentException( "values may not be null" );
    173
    174    this.operationDeclaredFields = operationDeclaredFields;
    175    this.outGroupingFields = asDeclaration( outGroupingFields );
    176    this.keySelectors = keySelectors;
    177    this.sortingSelectors = sortingSelectors; // null ok
    178    this.outValuesFields = asDeclaration( outValuesFields );
    179    }
    180
    181  /**
    182   * Constructor Scope creates a new Scope instance.
    183   *
    184   * @param name of type String
    185   */
    186  public Scope( String name )
    187    {
    188    this.name = name;
    189    }
    190
    191  public void addPriorNames( Scope incoming, Scope outgoing )
    192    {
    193    priorNames.addAll( 0, outgoing.priorNames );
    194    priorNames.addFirst( incoming.getName() );
    195    priorNames.addAll( 0, incoming.priorNames );
    196    }
    197
    198  public String getPriorName()
    199    {
    200    if( priorNames.isEmpty() )
    201      return getName();
    202
    203    return priorNames.getFirst();
    204    }
    205
    206  public Integer getOrdinal()
    207    {
    208    return ordinal;
    209    }
    210
    211  public void setOrdinal( Integer ordinal )
    212    {
    213    this.ordinal = ordinal;
    214    }
    215
    216  public void setNonBlocking( boolean isNonBlocking )
    217    {
    218    this.isNonBlocking = isNonBlocking;
    219    }
    220
    221  public boolean isNonBlocking()
    222    {
    223    return isNonBlocking;
    224    }
    225
    226  public boolean isSplice()
    227    {
    228    return isGroupBy() || isCoGroup() || isMerge() || isHashJoin();
    229    }
    230
    231  public boolean isGroup()
    232    {
    233    return kind == Kind.GROUPBY || kind == Kind.COGROUP;
    234    }
    235
    236  public boolean isGroupBy()
    237    {
    238    return kind == Kind.GROUPBY;
    239    }
    240
    241  public boolean isCoGroup()
    242    {
    243    return kind == Kind.COGROUP;
    244    }
    245
    246  public boolean isMerge()
    247    {
    248    return kind == Kind.MERGE;
    249    }
    250
    251  public boolean isHashJoin()
    252    {
    253    return kind == Kind.HASHJOIN;
    254    }
    255
    256  /**
    257   * Method isEach returns true if this Scope object represents an Each.
    258   *
    259   * @return the each (type boolean) of this Scope object.
    260   */
    261  public boolean isEach()
    262    {
    263    return kind == Kind.EACH;
    264    }
    265
    266  /**
    267   * Method isEvery returns true if this Scope object represents an Every.
    268   *
    269   * @return the every (type boolean) of this Scope object.
    270   */
    271  public boolean isEvery()
    272    {
    273    return kind == Kind.EVERY;
    274    }
    275
    276  /**
    277   * Method isTap returns true if this Scope object represents a Tap.
    278   *
    279   * @return the tap (type boolean) of this Scope object.
    280   */
    281  public boolean isTap()
    282    {
    283    return kind == Kind.TAP;
    284    }
    285
    286  /**
    287   * Method getName returns the name of this Scope object.
    288   *
    289   * @return the name (type String) of this Scope object.
    290   */
    291  public String getName()
    292    {
    293    return name;
    294    }
    295
    296  /**
    297   * Method setName sets the name of this Scope object.
    298   *
    299   * @param name the name of this Scope object.
    300   */
    301  public void setName( String name )
    302    {
    303    this.name = name;
    304    }
    305
    306  /**
    307   * Method getRemainderFields returns the remainderFields of this Scope object.
    308   *
    309   * @return the remainderFields (type Fields) of this Scope object.
    310   */
    311  public Fields getRemainderPassThroughFields()
    312    {
    313    return remainderPassThroughFields;
    314    }
    315
    316  /**
    317   * Method getArgumentSelector returns the argumentSelector of this Scope object.
    318   *
    319   * @return the argumentSelector (type Fields) of this Scope object.
    320   */
    321  public Fields getArgumentsSelector()
    322    {
    323    return operationArgumentFields;
    324    }
    325
    326  /**
    327   * Method getArguments returns the arguments of this Scope object.
    328   *
    329   * @return the arguments (type Fields) of this Scope object.
    330   */
    331  public Fields getArgumentsDeclarator()
    332    {
    333    return asDeclaration( operationArgumentFields );
    334    }
    335
    336  /**
    337   * Method getDeclaredFields returns the declaredFields of this Scope object.
    338   *
    339   * @return the declaredFields (type Fields) of this Scope object.
    340   */
    341  public Fields getOperationDeclaredFields()
    342    {
    343    return operationDeclaredFields;
    344    }
    345
    346  /**
    347   * Method getGroupingSelectors returns the groupingSelectors of this Scope object.
    348   *
    349   * @return the groupingSelectors (type Map<String, Fields>) of this Scope object.
    350   */
    351  public Map<String, Fields> getKeySelectors()
    352    {
    353    return keySelectors;
    354    }
    355
    356  /**
    357   * Method getSortingSelectors returns the sortingSelectors of this Scope object.
    358   *
    359   * @return the sortingSelectors (type Map<String, Fields>) of this Scope object.
    360   */
    361  public Map<String, Fields> getSortingSelectors()
    362    {
    363    return sortingSelectors;
    364    }
    365
    366  /**
    367   * Method getOutGroupingSelector returns the outGroupingSelector of this Scope object.
    368   *
    369   * @return the outGroupingSelector (type Fields) of this Scope object.
    370   */
    371  public Fields getOutGroupingSelector()
    372    {
    373    return outGroupingSelector;
    374    }
    375
    376  public Fields getIncomingTapFields()
    377    {
    378    if( isEvery() )
    379      return getOutGroupingFields();
    380    else
    381      return getOutValuesFields();
    382    }
    383
    384  public Fields getIncomingFunctionArgumentFields()
    385    {
    386    if( isEvery() )
    387      return getOutGroupingFields();
    388    else
    389      return getOutValuesFields();
    390    }
    391
    392  public Fields getIncomingFunctionPassThroughFields()
    393    {
    394    if( isEvery() )
    395      return getOutGroupingFields();
    396    else
    397      return getOutValuesFields();
    398    }
    399
    400  public Fields getIncomingAggregatorArgumentFields()
    401    {
    402    if( isEach() || isTap() )
    403      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    404
    405    return getOutValuesFields();
    406    }
    407
    408  public Fields getIncomingAggregatorPassThroughFields()
    409    {
    410    if( isEach() || isTap() )
    411      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    412
    413    return getOutGroupingFields();
    414    }
    415
    416  public Fields getIncomingBufferArgumentFields()
    417    {
    418    if( isEach() || isTap() )
    419      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    420
    421    return getOutValuesFields();
    422    }
    423
    424  public Fields getIncomingBufferPassThroughFields()
    425    {
    426    if( isEach() || isTap() )
    427      throw new IllegalStateException( "Every cannot follow a Tap or an Each" );
    428
    429    return getOutValuesFields();
    430    }
    431
    432  public Fields getIncomingSpliceFields()
    433    {
    434    if( isEvery() )
    435      return getOutGroupingFields();
    436    else
    437      return getOutValuesFields();
    438    }
    439
    440  /**
    441   * Method getOutGroupingFields returns the outGroupingFields of this Scope object.
    442   *
    443   * @return the outGroupingFields (type Fields) of this Scope object.
    444   */
    445  public Fields getOutGroupingFields()
    446    {
    447    if( !isSplice() )
    448      return outGroupingFields;
    449
    450    Fields first = keySelectors.values().iterator().next();
    451
    452    // if more than one, this is a merge, so same key names are expected
    453    if( keySelectors.size() == 1 || isGroupBy() )
    454      return first;
    455
    456    // handling CoGroup only
    457
    458    // if given by user as resultGroupFields
    459    if( outGroupingFields != null )
    460      return outGroupingFields;
    461
    462    // todo throw an exception if we make it this far
    463
    464    // if all have the same names, then use for grouping
    465    Set<Fields> set = new HashSet<Fields>( keySelectors.values() );
    466
    467    if( set.size() == 1 )
    468      return first;
    469
    470    return Fields.size( first.size() );
    471    }
    472
    473  public Fields getOutGroupingValueFields()
    474    {
    475    return getOutValuesFields().subtract( getOutGroupingFields() );
    476    }
    477
    478  /**
    479   * Method getOutValuesSelector returns the outValuesSelector of this Scope object.
    480   *
    481   * @return the outValuesSelector (type Fields) of this Scope object.
    482   */
    483  public Fields getOutValuesSelector()
    484    {
    485    return outValuesSelector;
    486    }
    487
    488  /**
    489   * Method getOutValuesFields returns the outValuesFields of this Scope object.
    490   *
    491   * @return the outValuesFields (type Fields) of this Scope object.
    492   */
    493  public Fields getOutValuesFields()
    494    {
    495    return outValuesFields;
    496    }
    497
    498  /**
    499   * Method copyFields copies the given Scope instance fields to this instance.
    500   *
    501   * @param scope of type Scope
    502   */
    503  public void copyFields( Scope scope )
    504    {
    505    this.kind = scope.kind;
    506    this.incomingPassThroughFields = scope.incomingPassThroughFields;
    507    this.remainderPassThroughFields = scope.remainderPassThroughFields;
    508    this.operationArgumentFields = scope.operationArgumentFields;
    509    this.operationDeclaredFields = scope.operationDeclaredFields;
    510    this.keySelectors = scope.keySelectors;
    511    this.sortingSelectors = scope.sortingSelectors;
    512    this.outGroupingSelector = scope.outGroupingSelector;
    513    this.outGroupingFields = scope.outGroupingFields;
    514    this.outValuesSelector = scope.outValuesSelector;
    515    this.outValuesFields = scope.outValuesFields;
    516    }
    517
    518  public String printSimple()
    519    {
    520    StringBuilder buffer = new StringBuilder();
    521
    522    if( name != null )
    523      buffer.append( "[" ).append( name ).append( "]" );
    524
    525    buffer.append( "{id=" ).append( System.identityHashCode( this ) );
    526
    527    if( ordinal != null )
    528      buffer.append( ",ordinal=" ).append( ordinal );
    529
    530    if( kind != null )
    531      buffer.append( ",kind=" ).append( kind );
    532
    533    buffer.append( "}" );
    534
    535    return buffer.toString();
    536    }
    537
    538  @Override
    539  public String toString()
    540    {
    541    return print();
    542    }
    543
    544  public String print()
    545    {
    546    StringBuilder buffer = new StringBuilder();
    547
    548    if( ordinal != null )
    549      buffer.append( "[" ).append( ordinal ).append( "]" );
    550
    551    if( getOutValuesFields() == null )
    552      return buffer.toString(); // endpipes
    553
    554    buffer.append( "\n" );
    555
    556    if( keySelectors != null && !keySelectors.isEmpty() )
    557      {
    558      for( String name : keySelectors.keySet() )
    559        {
    560        if( buffer.length() != 0 && buffer.charAt( buffer.length() - 1 ) != '\n' )
    561          buffer.append( " | " );
    562
    563        buffer.append( name ).append( keySelectors.get( name ).printVerbose() );
    564        }
    565
    566      buffer.append( "\n" );
    567      }
    568
    569    if( outGroupingFields != null )
    570      buffer.append( getOutGroupingFields().printVerbose() ).append( "\n" );
    571
    572    buffer.append( getOutValuesFields().printVerbose() );
    573
    574    return buffer.toString();
    575    }
    576  }