001/*
002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
004 *
005 * Project and contact information: http://www.cascading.org/
006 *
007 * This file is part of the Cascading project.
008 *
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *     http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 */
021
022package cascading.tuple;
023
024import java.beans.ConstructorProperties;
025import java.io.Serializable;
026import java.lang.reflect.Type;
027import java.util.ArrayList;
028import java.util.Arrays;
029import java.util.Collections;
030import java.util.Comparator;
031import java.util.HashMap;
032import java.util.HashSet;
033import java.util.Iterator;
034import java.util.LinkedHashSet;
035import java.util.LinkedList;
036import java.util.List;
037import java.util.Map;
038import java.util.Set;
039import java.util.stream.IntStream;
040
041import cascading.tap.Tap;
042import cascading.tuple.type.CoercibleType;
043import cascading.util.Util;
044
045/**
046 * Class Fields represents the field names in a {@link Tuple}. A tuple field may be a literal String value representing a
047 * name, or it may be a literal Integer value representing a position, where positions start at position 0.
048 * A Fields instance may also represent a set of field names and positions.
049 * <p>
050 * Fields are used as both declarators and selectors. A declarator declares that a given {@link Tap} or
051 * {@link cascading.operation.Operation} returns the given field names, for a set of values the size of
052 * the given Fields instance. A selector is used to select given referenced fields from a Tuple.
053 * For example; <br>
054 * {@code Fields fields = new Fields( "a", "b", "c" );}<br>
055 * This creates a new Fields instance with the field names "a", "b", and "c". This Fields instance can be used as both
056 * a declarator or a selector, depending on how it's used.
057 * <p>
058 * Or For example; <br>
059 * {@code Fields fields = new Fields( 1, 2, -1 );}<br>
060 * This creates a new Fields instance that can only be used as a selector. It would select the second, third, and last
061 * position from a given Tuple instance, assuming it has at least four positions. Since the original field names for those
062 * positions will carry over to the new selected Tuple instance, if the original Tuple only had three positions, the third
063 * and last positions would be the same, and would throw an error on there being duplicate field names in the selected
064 * Tuple instance.
065 * <p>
066 * Additionally, there are eight predefined Fields sets used for different purposes; {@link #NONE}, {@link #ALL}, {@link #GROUP},
067 * {@link #VALUES}, {@link #ARGS}, {@link #RESULTS}, {@link #UNKNOWN}, {@link #REPLACE}, and {@link #SWAP}.
068 * <p>
069 * The {@code NONE} Fields set represents no fields.
070 * <p>
071 * The {@code ALL} Fields set is a "wildcard" that represents all the current available fields.
072 * <p>
073 * The {@code GROUP} Fields set represents all the fields used as grouping values in a previous {@link cascading.pipe.Splice}.
074 * If there is no previous Group in the pipe assembly, the GROUP represents all the current field names.
075 * <p>
076 * The {@code VALUES} Fields set represent all the fields not used as grouping fields in a previous Group.
077 * <p>
078 * The {@code ARGS} Fields set is used to let a given Operation inherit the field names of its argument Tuple. This Fields set
079 * is a convenience and is typically used when the Pipe output selector is {@code RESULTS} or {@code REPLACE}.
080 * <p>
081 * The {@code RESULTS} Fields set is used to represent the field names of the current Operations return values. This Fields
082 * set may only be used as an output selector on a Pipe. It effectively replaces in the input Tuple with the Operation result
083 * Tuple.
084 * <p>
085 * The {@code UNKNOWN} Fields set is used when Fields must be declared, but how many and their names is unknown. This allows
086 * for arbitrarily length Tuples from an input source or some Operation. Use this Fields set with caution.
087 * <p>
088 * The {@code REPLACE} Fields set is used as an output selector to inline replace values in the incoming Tuple with
089 * the results of an Operation. This is a convenience Fields set that allows subsequent Operations to 'step' on the
090 * value with a given field name. The current Operation must always use the exact same field names, or the {@code ARGS}
091 * Fields set.
092 * <p>
093 * The {@code SWAP} Fields set is used as an output selector to swap out Operation arguments with its results. Neither
094 * the argument and result field names or size need to be the same. This is useful for when the Operation arguments are
095 * no longer necessary and the result Fields and values should be appended to the remainder of the input field names
096 * and Tuple.
097 */
098public class Fields implements Comparable, Iterable<Comparable>, Serializable, Comparator<Tuple>
099  {
100  /** Field UNKNOWN */
101  public static final Fields UNKNOWN = new Fields( Kind.UNKNOWN );
102  /** Field NONE represents a wildcard for no fields */
103  public static final Fields NONE = new Fields( Kind.NONE );
104  /** Field ALL represents a wildcard for all fields */
105  public static final Fields ALL = new Fields( Kind.ALL );
106  /** Field KEYS represents all fields used as they key for the last grouping */
107  public static final Fields GROUP = new Fields( Kind.GROUP );
108  /** Field VALUES represents all fields used as values for the last grouping */
109  public static final Fields VALUES = new Fields( Kind.VALUES );
110  /** Field ARGS represents all fields used as the arguments for the current operation */
111  public static final Fields ARGS = new Fields( Kind.ARGS );
112  /** Field RESULTS represents all fields returned by the current operation */
113  public static final Fields RESULTS = new Fields( Kind.RESULTS );
114  /** Field REPLACE represents all incoming fields, and allows their values to be replaced by the current operation results. */
115  public static final Fields REPLACE = new Fields( Kind.REPLACE );
116  /** Field SWAP represents all fields not used as arguments for the current operation and the operations results. */
117  public static final Fields SWAP = new Fields( Kind.SWAP );
118  /** Field FIRST represents the first field position, 0 */
119  public static final Fields FIRST = new Fields( 0 );
120  /** Field LAST represents the last field position, -1 */
121  public static final Fields LAST = new Fields( -1 );
122
123  /** Field EMPTY_INT */
124  private static final int[] EMPTY_INT = new int[ 0 ];
125
126  /**
127   */
128  enum Kind
129    {
130      NONE, ALL, GROUP, VALUES, ARGS, RESULTS, UNKNOWN, REPLACE, SWAP
131    }
132
133  /** Field fields */
134  Comparable[] fields = new Comparable[ 0 ];
135  /** Field isOrdered */
136  boolean isOrdered = true;
137  /** Field kind */
138  Kind kind;
139
140  /** Field types */
141  Type[] types;
142  /** Field comparators */
143  Comparator[] comparators;
144
145  /** Field thisPos */
146  transient int[] thisPos;
147  /** Field index */
148  transient Map<Comparable, Integer> index;
149  /** Field posCache */
150  transient Map<Fields, int[]> posCache;
151  /** Field hashCode */
152  transient int hashCode; // need to cache this
153
154  /**
155   * Method fields is a convenience method to create an array of Fields instances.
156   *
157   * @param fields of type Fields
158   * @return Fields[]
159   */
160  public static Fields[] fields( Fields... fields )
161    {
162    return fields;
163    }
164
165  public static Comparable[] names( Comparable... names )
166    {
167    return names;
168    }
169
170  public static Type[] types( Type... types )
171    {
172    return types;
173    }
174
175  /**
176   * Method size is a factory that makes new instances of Fields the given size.
177   *
178   * @param size of type int
179   * @return Fields
180   */
181  public static Fields size( int size )
182    {
183    if( size == 0 )
184      return Fields.NONE;
185
186    Fields fields = new Fields();
187
188    fields.kind = null;
189    fields.fields = expand( size, 0 );
190
191    return fields;
192    }
193
194  /**
195   * Method size is a factory that makes new instances of Fields the given size with every field
196   * of the given type.
197   *
198   * @param size of type int
199   * @param type of type Type
200   * @return Fields
201   */
202  public static Fields size( int size, Type type )
203    {
204    if( size == 0 )
205      return Fields.NONE;
206
207    Fields fields = new Fields();
208
209    fields.kind = null;
210    fields.fields = expand( size, 0 );
211
212    for( Comparable field : fields )
213      fields = fields.applyType( field, type );
214
215    return fields;
216    }
217
218  /**
219   * Method join joins all given Fields instances into a new Fields instance.
220   * <p>
221   * Use caution with this method, it does not assume the given Fields are either selectors or declarators. Numeric position fields are left untouched.
222   * <p>
223   * If the resulting set of fields and ordinals is length zero, {@link Fields#NONE} will be returned.
224   *
225   * @param fields of type Fields
226   * @return Fields
227   */
228  public static Fields join( Fields... fields )
229    {
230    return join( false, fields );
231    }
232
233  public static Fields join( boolean maskDuplicateNames, Fields... fields )
234    {
235    int size = 0;
236
237    for( Fields field : fields )
238      {
239      if( field.isSubstitution() || field.isUnknown() )
240        throw new TupleException( "cannot join fields if one is a substitution or is unknown" );
241
242      size += field.size();
243      }
244
245    if( size == 0 )
246      return Fields.NONE;
247
248    Comparable[] elements = join( size, fields );
249
250    if( maskDuplicateNames )
251      {
252      Set<String> names = new HashSet<String>();
253
254      for( int i = elements.length - 1; i >= 0; i-- )
255        {
256        Comparable element = elements[ i ];
257
258        if( names.contains( element ) )
259          elements[ i ] = i;
260        else if( element instanceof String )
261          names.add( (String) element );
262        }
263      }
264
265    Type[] types = joinTypes( size, fields );
266
267    if( types == null )
268      return new Fields( elements );
269    else
270      return new Fields( elements, types );
271    }
272
273  private static Comparable[] join( int size, Fields... fields )
274    {
275    Comparable[] elements = expand( size, 0 );
276
277    int pos = 0;
278    for( Fields field : fields )
279      {
280      System.arraycopy( field.fields, 0, elements, pos, field.size() );
281      pos += field.size();
282      }
283
284    return elements;
285    }
286
287  private static Type[] joinTypes( int size, Fields... fields )
288    {
289    Type[] elements = new Type[ size ];
290
291    int pos = 0;
292    for( Fields field : fields )
293      {
294      if( field.isNone() )
295        continue;
296
297      if( field.types == null )
298        return null;
299
300      System.arraycopy( field.types, 0, elements, pos, field.size() );
301      pos += field.size();
302      }
303
304    return elements;
305    }
306
307  public static Fields mask( Fields fields, Fields mask )
308    {
309    Comparable[] elements = expand( fields.size(), 0 );
310
311    System.arraycopy( fields.fields, 0, elements, 0, elements.length );
312
313    for( int i = elements.length - 1; i >= 0; i-- )
314      {
315      Comparable element = elements[ i ];
316
317      if( element instanceof Integer )
318        continue;
319
320      if( mask.getIndex().containsKey( element ) )
321        elements[ i ] = i;
322      }
323
324    return new Fields( elements );
325    }
326
327  /**
328   * Method merge merges all given Fields instances into a new Fields instance where a merge is a set union of all the
329   * given Fields instances.
330   * <p>
331   * Thus duplicate positions or field names are allowed, they are subsequently discarded in favor of the first
332   * occurrence. That is, merging "a" and "a" would yield "a", not "a, a", yet merging "a,b" and "c" would yield "a,b,c".
333   * <p>
334   * Use caution with this method, it does not assume the given Fields are either selectors or declarators. Numeric position fields are left untouched.
335   *
336   * @param fields of type Fields
337   * @return Fields
338   */
339  public static Fields merge( Fields... fields )
340    {
341    List<Comparable> elements = new ArrayList<Comparable>();
342    List<Type> elementTypes = new ArrayList<Type>();
343
344    for( Fields field : fields )
345      {
346      Type[] types = field.getTypes();
347      int i = 0;
348
349      for( Comparable comparable : field )
350        {
351        if( !elements.contains( comparable ) )
352          {
353          elements.add( comparable );
354          elementTypes.add( types == null ? null : types[ i ] ); // nulls ok
355          }
356
357        i++;
358        }
359      }
360
361    Comparable[] comparables = elements.toArray( new Comparable[ elements.size() ] );
362    Type[] types = elementTypes.toArray( new Type[ elementTypes.size() ] );
363
364    if( Util.containsNull( types ) )
365      return new Fields( comparables );
366
367    return new Fields( comparables, types );
368    }
369
370  public static Fields copyComparators( Fields toFields, Fields... fromFields )
371    {
372    for( Fields fromField : fromFields )
373      {
374      for( Comparable field : fromField )
375        {
376        Comparator comparator = fromField.getComparator( field );
377
378        if( comparator != null )
379          toFields.setComparator( field, comparator );
380        }
381      }
382
383    return toFields;
384    }
385
386  /**
387   * Method offsetSelector is a factory that makes new instances of Fields the given size but offset by startPos.
388   * The result Fields instance can only be used as a selector.
389   *
390   * @param size     of type int
391   * @param startPos of type int
392   * @return Fields
393   */
394  public static Fields offsetSelector( int size, int startPos )
395    {
396    Fields fields = new Fields();
397
398    fields.kind = null;
399    fields.isOrdered = startPos == 0;
400    fields.fields = expand( size, startPos );
401
402    return fields;
403    }
404
405  private static Comparable[] expand( int size, int startPos )
406    {
407    if( size < 1 )
408      throw new TupleException( "invalid size for fields: " + size );
409
410    if( startPos < 0 )
411      throw new TupleException( "invalid start position for fields: " + startPos );
412
413    Comparable[] fields = new Comparable[ size ];
414
415    for( int i = 0; i < fields.length; i++ )
416      fields[ i ] = i + startPos;
417
418    return fields;
419    }
420
421  /**
422   * Method resolve returns a new selector expanded on the given field declarations
423   *
424   * @param selector of type Fields
425   * @param fields   of type Fields
426   * @return Fields
427   */
428  public static Fields resolve( Fields selector, Fields... fields )
429    {
430    boolean hasUnknowns = false;
431    int size = 0;
432
433    for( Fields field : fields )
434      {
435      if( field.isUnknown() )
436        hasUnknowns = true;
437
438      if( !field.isDefined() && field.isUnOrdered() )
439        throw new TupleException( "unable to select from field set: " + field.printVerbose() );
440
441      size += field.size();
442      }
443
444    if( selector.isAll() )
445      {
446      Fields result = fields[ 0 ];
447
448      for( int i = 1; i < fields.length; i++ )
449        result = result.append( fields[ i ] );
450
451      return result;
452      }
453
454    if( selector.isReplace() )
455      {
456      if( fields[ 1 ].isUnknown() )
457        throw new TupleException( "cannot replace fields with unknown field declaration" );
458
459      if( !fields[ 0 ].contains( fields[ 1 ] ) )
460        throw new TupleException( "could not find all fields to be replaced, available: " + fields[ 0 ].printVerbose() + ",  declared: " + fields[ 1 ].printVerbose() );
461
462      Type[] types = fields[ 0 ].getTypes();
463
464      if( types != null )
465        {
466        for( int i = 1; i < fields.length; i++ )
467          {
468          Type[] fieldTypes = fields[ i ].getTypes();
469
470          if( fieldTypes == null )
471            {
472            fields[ 0 ] = fields[ 0 ].applyTypes( (Type[]) null );
473            }
474          else
475            {
476            for( int j = 0; j < fieldTypes.length; j++ )
477              fields[ 0 ] = fields[ 0 ].applyType( fields[ i ].get( j ), fieldTypes[ j ] );
478            }
479          }
480        }
481
482      return fields[ 0 ];
483      }
484
485    // we can't deal with anything but ALL
486    if( !selector.isDefined() )
487      throw new TupleException( "unable to use given selector: " + selector );
488
489    Set<String> notFound = new LinkedHashSet<String>();
490    Set<String> found = new HashSet<String>();
491    Fields result = size( selector.size() );
492
493    if( hasUnknowns )
494      size = -1;
495
496    Type[] types = null;
497
498    if( size != -1 )
499      types = new Type[ result.size() ];
500
501    int offset = 0;
502    for( Fields current : fields )
503      {
504      if( current.isNone() )
505        continue;
506
507      resolveInto( notFound, found, selector, current, result, types, offset, size );
508      offset += current.size();
509      }
510
511    if( types != null && !Util.containsNull( types ) ) // don't apply types if any are null
512      result = result.applyTypes( types );
513
514    notFound.removeAll( found );
515
516    if( !notFound.isEmpty() )
517      throw new FieldsResolverException( new Fields( join( size, fields ) ), new Fields( notFound.toArray( new Comparable[ notFound.size() ] ) ) );
518
519    if( hasUnknowns )
520      return selector;
521
522    return result;
523    }
524
525  private static void resolveInto( Set<String> notFound, Set<String> found, Fields selector, Fields current, Fields result, Type[] types, int offset, int size )
526    {
527    for( int i = 0; i < selector.size(); i++ )
528      {
529      Comparable field = selector.get( i );
530
531      if( field instanceof String )
532        {
533        int index = current.indexOfSafe( field );
534
535        if( index == -1 )
536          notFound.add( (String) field );
537        else
538          result.set( i, handleFound( found, field ) );
539
540        if( index != -1 && types != null && current.getType( index ) != null )
541          types[ i ] = current.getType( index );
542
543        continue;
544        }
545
546      int pos = current.translatePos( (Integer) field, size ) - offset;
547
548      if( pos >= current.size() || pos < 0 )
549        continue;
550
551      Comparable thisField = current.get( pos );
552
553      if( types != null && current.getType( pos ) != null )
554        types[ i ] = current.getType( pos );
555
556      if( thisField instanceof String )
557        result.set( i, handleFound( found, thisField ) );
558      else
559        result.set( i, field );
560      }
561    }
562
563  private static Comparable handleFound( Set<String> found, Comparable field )
564    {
565    if( found.contains( field ) )
566      throw new TupleException( "field name already exists: " + field );
567
568    found.add( (String) field );
569
570    return field;
571    }
572
573  /**
574   * Method asDeclaration returns a new Fields instance for use as a declarator based on the given fields value.
575   * <p>
576   * Typically this is used to convert a selector to a declarator. Simply, all numeric position fields are replaced
577   * by their absolute position.
578   * <p>
579   * Comparators are preserved in the result.
580   *
581   * @param fields of type Fields
582   * @return Fields
583   */
584  public static Fields asDeclaration( Fields fields )
585    {
586    if( fields == null )
587      return null;
588
589    if( fields.isNone() )
590      return fields;
591
592    if( !fields.isDefined() )
593      return UNKNOWN;
594
595    if( fields.isOrdered() )
596      return fields;
597
598    Fields result = size( fields.size() );
599
600    copy( null, result, fields, 0 );
601
602    result.types = copyTypes( fields.types, result.size() );
603    result.comparators = fields.comparators;
604
605    return result;
606    }
607
608  private static Fields asSelector( Fields fields )
609    {
610    if( !fields.isDefined() )
611      return UNKNOWN;
612
613    return fields;
614    }
615
616  /**
617   * Constructor Fields creates a new Fields instance.
618   *
619   * @param kind of type Kind
620   */
621  protected Fields( Kind kind )
622    {
623    this.kind = kind;
624    }
625
626  public Fields()
627    {
628    this.kind = Kind.NONE;
629    }
630
631  /**
632   * Constructor Fields creates a new Fields instance.
633   *
634   * @param fields of type Comparable...
635   */
636  @ConstructorProperties({"fields"})
637  public Fields( Comparable... fields )
638    {
639    if( fields.length == 0 )
640      this.kind = Kind.NONE;
641    else
642      this.fields = validate( fields );
643    }
644
645  @ConstructorProperties({"field", "type"})
646  public Fields( Comparable field, Type type )
647    {
648    this( names( field ), types( type ) );
649    }
650
651  @ConstructorProperties({"fields", "types"})
652  public Fields( Comparable[] fields, Type[] types )
653    {
654    this( fields );
655
656    if( isDefined() && types != null )
657      {
658      if( this.fields.length != types.length )
659        throw new IllegalArgumentException( "given types array must be same length as fields" );
660
661      if( Util.containsNull( types ) )
662        throw new IllegalArgumentException( "given types array contains null" );
663
664      this.types = copyTypes( types, this.fields.length );
665      }
666    }
667
668  @ConstructorProperties({"types"})
669  public Fields( Type... types )
670    {
671    if( types.length == 0 )
672      {
673      this.kind = Kind.NONE;
674      return;
675      }
676
677    this.fields = expand( types.length, 0 );
678
679    if( this.fields.length != types.length )
680      throw new IllegalArgumentException( "given types array must be same length as fields" );
681
682    if( Util.containsNull( types ) )
683      throw new IllegalArgumentException( "given types array contains null" );
684
685    this.types = copyTypes( types, this.fields.length );
686    }
687
688  /**
689   * Method isUnOrdered returns true if this instance is unordered. That is, it has relative numeric field positions.
690   * For example; [1,"a",2,-1]
691   *
692   * @return the unOrdered (type boolean) of this Fields object.
693   */
694  public boolean isUnOrdered()
695    {
696    return !isOrdered || kind == Kind.ALL;
697    }
698
699  /**
700   * Method isOrdered returns true if this instance is ordered. That is, all numeric field positions are absolute.
701   * For example; [0,"a",2,3]
702   *
703   * @return the ordered (type boolean) of this Fields object.
704   */
705  public boolean isOrdered()
706    {
707    return isOrdered || kind == Kind.UNKNOWN;
708    }
709
710  /**
711   * Method isDefined returns true if this instance is not a field set like {@link #ALL} or {@link #UNKNOWN}.
712   *
713   * @return the defined (type boolean) of this Fields object.
714   */
715  public boolean isDefined()
716    {
717    return kind == null;
718    }
719
720  /**
721   * Method isOutSelector returns true if this instance is 'defined', or the field set {@link #ALL} or {@link #RESULTS}.
722   *
723   * @return the outSelector (type boolean) of this Fields object.
724   */
725  public boolean isOutSelector()
726    {
727    return isAll() || isResults() || isReplace() || isSwap() || isDefined();
728    }
729
730  /**
731   * Method isArgSelector returns true if this instance is 'defined' or the field set {@link #ALL}, {@link #GROUP}, or
732   * {@link #VALUES}.
733   *
734   * @return the argSelector (type boolean) of this Fields object.
735   */
736  public boolean isArgSelector()
737    {
738    return isAll() || isNone() || isGroup() || isValues() || isDefined();
739    }
740
741  /**
742   * Method isDeclarator returns true if this can be used as a declarator. Specifically if it is 'defined' or
743   * {@link #UNKNOWN}, {@link #ALL}, {@link #ARGS}, {@link #GROUP}, or {@link #VALUES}.
744   *
745   * @return the declarator (type boolean) of this Fields object.
746   */
747  public boolean isDeclarator()
748    {
749    return isUnknown() || isNone() || isAll() || isArguments() || isGroup() || isValues() || isDefined();
750    }
751
752  /**
753   * Method isNone returns returns true if this instance is the {@link #NONE} field set.
754   *
755   * @return the none (type boolean) of this Fields object.
756   */
757  public boolean isNone()
758    {
759    return kind == Kind.NONE;
760    }
761
762  /**
763   * Method isAll returns true if this instance is the {@link #ALL} field set.
764   *
765   * @return the all (type boolean) of this Fields object.
766   */
767  public boolean isAll()
768    {
769    return kind == Kind.ALL;
770    }
771
772  /**
773   * Method isUnknown returns true if this instance is the {@link #UNKNOWN} field set.
774   *
775   * @return the unknown (type boolean) of this Fields object.
776   */
777  public boolean isUnknown()
778    {
779    return kind == Kind.UNKNOWN;
780    }
781
782  /**
783   * Method isArguments returns true if this instance is the {@link #ARGS} field set.
784   *
785   * @return the arguments (type boolean) of this Fields object.
786   */
787  public boolean isArguments()
788    {
789    return kind == Kind.ARGS;
790    }
791
792  /**
793   * Method isValues returns true if this instance is the {@link #VALUES} field set.
794   *
795   * @return the values (type boolean) of this Fields object.
796   */
797  public boolean isValues()
798    {
799    return kind == Kind.VALUES;
800    }
801
802  /**
803   * Method isResults returns true if this instance is the {@link #RESULTS} field set.
804   *
805   * @return the results (type boolean) of this Fields object.
806   */
807  public boolean isResults()
808    {
809    return kind == Kind.RESULTS;
810    }
811
812  /**
813   * Method isReplace returns true if this instance is the {@link #REPLACE} field set.
814   *
815   * @return the replace (type boolean) of this Fields object.
816   */
817  public boolean isReplace()
818    {
819    return kind == Kind.REPLACE;
820    }
821
822  /**
823   * Method isSwap returns true if this instance is the {@link #SWAP} field set.
824   *
825   * @return the swap (type boolean) of this Fields object.
826   */
827  public boolean isSwap()
828    {
829    return kind == Kind.SWAP;
830    }
831
832  /**
833   * Method isKeys returns true if this instance is the {@link #GROUP} field set.
834   *
835   * @return the keys (type boolean) of this Fields object.
836   */
837  public boolean isGroup()
838    {
839    return kind == Kind.GROUP;
840    }
841
842  /**
843   * Method isSubstitution returns true if this instance is a substitution fields set. Specifically if it is the field
844   * set {@link #ALL}, {@link #ARGS}, {@link #GROUP}, or {@link #VALUES}.
845   *
846   * @return the substitution (type boolean) of this Fields object.
847   */
848  public boolean isSubstitution()
849    {
850    return isAll() || isArguments() || isGroup() || isValues();
851    }
852
853  private Comparable[] validate( Comparable[] fields )
854    {
855    isOrdered = true;
856
857    Set<Comparable> names = new HashSet<Comparable>();
858
859    for( int i = 0; i < fields.length; i++ )
860      {
861      Comparable field = fields[ i ];
862
863      if( !( field instanceof String || field instanceof Integer ) )
864        throw new IllegalArgumentException( String.format( "invalid field type (%s); must be String or Integer: ", field ) );
865
866      if( names.contains( field ) )
867        throw new IllegalArgumentException( "duplicate field name found: " + field );
868
869      names.add( field );
870
871      if( field instanceof Number && (Integer) field != i )
872        isOrdered = false;
873      }
874
875    return fields;
876    }
877
878  final Comparable[] get()
879    {
880    return fields;
881    }
882
883  /**
884   * Method get returns the field name or position at the given index i.
885   *
886   * @param i is of type int
887   * @return Comparable
888   */
889  public final Comparable get( int i )
890    {
891    return fields[ i ];
892    }
893
894  final void set( int i, Comparable comparable )
895    {
896    fields[ i ] = comparable;
897
898    if( isOrdered() && comparable instanceof Integer )
899      isOrdered = i == (Integer) comparable;
900    }
901
902  /**
903   * Method getPos returns the pos array of this Fields object.
904   *
905   * @return the pos (type int[]) of this Fields object.
906   */
907  public int[] getPos()
908    {
909    if( thisPos != null )
910      return thisPos; // do not clone
911
912    if( isAll() || isUnknown() )
913      thisPos = EMPTY_INT;
914    else
915      thisPos = makeThisPos();
916
917    return thisPos;
918    }
919
920  /**
921   * Method hasRelativePos returns true if any ordinal position is relative (&lt; 0)
922   *
923   * @return true if any ordinal position is relative (&lt; 0)
924   */
925  public boolean hasRelativePos()
926    {
927    for( int i : getPos() )
928      {
929      if( i < 0 )
930        return true;
931      }
932
933    return false;
934    }
935
936  private int[] makeThisPos()
937    {
938    int[] pos = new int[ size() ];
939
940    for( int i = 0; i < size(); i++ )
941      {
942      Comparable field = get( i );
943
944      if( field instanceof Number )
945        pos[ i ] = (Integer) field;
946      else
947        pos[ i ] = i;
948      }
949
950    return pos;
951    }
952
953  private final Map<Fields, int[]> getPosCache()
954    {
955    if( posCache == null )
956      posCache = new HashMap<Fields, int[]>();
957
958    return posCache;
959    }
960
961  private final int[] putReturn( Fields fields, int[] pos )
962    {
963    getPosCache().put( fields, pos );
964
965    return pos;
966    }
967
968  public final int[] getPos( Fields fields )
969    {
970    return getPos( fields, -1 );
971    }
972
973  final int[] getPos( Fields fields, int tupleSize )
974    {
975    // test for key, as we stuff a null value
976    int[] pos = getPosCache().get( fields );
977
978    if( !isUnknown() && pos != null )
979      return pos;
980
981    if( fields.isAll() )
982      return putReturn( fields, null ); // return null, not getPos()
983
984    if( isAll() )
985      return putReturn( fields, fields.getPos() );
986
987    // don't cache unknown
988    if( size() == 0 && isUnknown() )
989      return translatePos( fields, tupleSize );
990
991    pos = translatePos( fields, size() );
992
993    return putReturn( fields, pos );
994    }
995
996  private int[] translatePos( Fields fields, int fieldSize )
997    {
998    int[] pos = new int[ fields.size() ];
999
1000    for( int i = 0; i < fields.size(); i++ )
1001      {
1002      Comparable field = fields.get( i );
1003
1004      if( field instanceof Number )
1005        pos[ i ] = translatePos( (Integer) field, fieldSize );
1006      else
1007        pos[ i ] = indexOf( field );
1008      }
1009
1010    return pos;
1011    }
1012
1013  final int translatePos( Integer integer )
1014    {
1015    return translatePos( integer, size() );
1016    }
1017
1018  final int translatePos( Integer integer, int size )
1019    {
1020    if( size == -1 )
1021      return integer;
1022
1023    if( integer < 0 )
1024      integer = size + integer;
1025
1026    if( !isUnknown() && ( integer >= size || integer < 0 ) )
1027      throw new TupleException( "position value is too large: " + integer + ", positions in field: " + size );
1028
1029    return integer;
1030    }
1031
1032  /**
1033   * Method getPos returns the index of the give field value in this Fields instance. The index corresponds to the
1034   * Tuple value index in an associated Tuple instance.
1035   *
1036   * @param fieldName of type Comparable
1037   * @return int
1038   */
1039  public int getPos( Comparable fieldName )
1040    {
1041    if( fieldName instanceof Number )
1042      return translatePos( (Integer) fieldName );
1043    else
1044      return indexOf( fieldName );
1045    }
1046
1047  private final Map<Comparable, Integer> getIndex()
1048    {
1049    if( index != null )
1050      return index;
1051
1052    // make thread-safe by not having invalid intermediate state
1053    Map<Comparable, Integer> local = new HashMap<Comparable, Integer>();
1054
1055    for( int i = 0; i < size(); i++ )
1056      local.put( get( i ), i );
1057
1058    return index = local;
1059    }
1060
1061  private int indexOf( Comparable fieldName )
1062    {
1063    Integer result = getIndex().get( fieldName );
1064
1065    if( result == null )
1066      throw new FieldsResolverException( this, new Fields( fieldName ) );
1067
1068    return result;
1069    }
1070
1071  int indexOfSafe( Comparable fieldName )
1072    {
1073    Integer result = getIndex().get( fieldName );
1074
1075    if( result == null )
1076      return -1;
1077
1078    return result;
1079    }
1080
1081  /**
1082   * Method iterator returns an unmodifiable iterator of field values. If {@link #isSubstitution()} returns true,
1083   * this iterator will be empty.
1084   *
1085   * @return Iterator of Comparable instances
1086   */
1087  public Iterator iterator()
1088    {
1089    return Arrays.stream( fields ).iterator();
1090    }
1091
1092  /**
1093   * Method fieldsIterator returns an iterator of Fields instances for each unique field declared by this Fields
1094   * instance. If {@link #isSubstitution()} returns true,
1095   * this iterator will be empty.
1096   *
1097   * @return Iterator of Fields instances
1098   */
1099  public Iterator<Fields> fieldsIterator()
1100    {
1101    if( types == null )
1102      return Arrays.stream( fields )
1103        .map( Fields::new )
1104        .iterator();
1105
1106    return IntStream.range( 0, fields.length )
1107      .mapToObj( pos -> new Fields( fields[ pos ], types[ pos ] ) )
1108      .iterator();
1109    }
1110
1111  /**
1112   * Method select returns a new Fields instance with fields specified by the given selector.
1113   *
1114   * @param selector of type Fields
1115   * @return Fields
1116   */
1117  public Fields select( Fields selector )
1118    {
1119    if( !isOrdered() )
1120      throw new TupleException( "this fields instance can only be used as a selector" );
1121
1122    if( selector.isAll() )
1123      return this;
1124
1125    // supports -1_UNKNOWN_RETURNED
1126    // guarantees pos arguments remain selector positions, not absolute positions
1127    if( isUnknown() )
1128      return asSelector( selector );
1129
1130    if( selector.isNone() )
1131      return NONE;
1132
1133    Fields result = size( selector.size() );
1134
1135    for( int i = 0; i < selector.size(); i++ )
1136      {
1137      Comparable field = selector.get( i );
1138
1139      if( field instanceof String )
1140        {
1141        result.set( i, get( indexOf( field ) ) );
1142        continue;
1143        }
1144
1145      int pos = translatePos( (Integer) field );
1146
1147      if( this.get( pos ) instanceof String )
1148        result.set( i, this.get( pos ) );
1149      else
1150        result.set( i, pos ); // use absolute position if no field name
1151      }
1152
1153    if( this.types != null )
1154      {
1155      result.types = new Type[ result.size() ];
1156
1157      for( int i = 0; i < selector.size(); i++ )
1158        {
1159        Comparable field = selector.get( i );
1160
1161        if( field instanceof String )
1162          result.setType( i, getType( indexOf( field ) ) );
1163        else
1164          result.setType( i, getType( translatePos( (Integer) field ) ) );
1165        }
1166      }
1167
1168    return result;
1169    }
1170
1171  /**
1172   * Method selectPos returns a Fields instance with only positional fields, no field names.
1173   *
1174   * @param selector of type Fields
1175   * @return Fields instance with only positions.
1176   */
1177  public Fields selectPos( Fields selector )
1178    {
1179    return selectPos( selector, 0 );
1180    }
1181
1182  /**
1183   * Method selectPos returns a Fields instance with only positional fields, offset by given offset value, no field names.
1184   *
1185   * @param selector of type Fields
1186   * @param offset   of type int
1187   * @return Fields instance with only positions.
1188   */
1189  public Fields selectPos( Fields selector, int offset )
1190    {
1191    int[] pos = getPos( selector );
1192
1193    Fields results = size( pos.length );
1194
1195    for( int i = 0; i < pos.length; i++ )
1196      results.fields[ i ] = pos[ i ] + offset;
1197
1198    return results;
1199    }
1200
1201  /**
1202   * Method subtract returns the difference between this instance and the given fields instance.
1203   * <p>
1204   * See {@link #append(Fields)} for adding field names.
1205   *
1206   * @param fields of type Fields
1207   * @return Fields
1208   */
1209  public Fields subtract( Fields fields )
1210    {
1211    if( fields.isAll() )
1212      return Fields.NONE;
1213
1214    if( fields.isNone() )
1215      return this;
1216
1217    List<Comparable> list = new LinkedList<Comparable>();
1218    Collections.addAll( list, this.get() );
1219    int[] pos = getPos( fields, -1 );
1220
1221    for( int i : pos )
1222      list.set( i, null );
1223
1224    Util.removeAllNulls( list );
1225
1226    Type[] newTypes = null;
1227
1228    if( this.types != null )
1229      {
1230      List<Type> types = new LinkedList<Type>();
1231      Collections.addAll( types, this.types );
1232
1233      for( int i : pos )
1234        types.set( i, null );
1235
1236      Util.removeAllNulls( types );
1237
1238      newTypes = types.toArray( new Type[ types.size() ] );
1239      }
1240
1241    return new Fields( list.toArray( new Comparable[ list.size() ] ), newTypes );
1242    }
1243
1244  /**
1245   * Method is used for appending the given Fields instance to this instance, into a new Fields instance suitable
1246   * for use as a field declaration.
1247   * <p>
1248   * That is, any positional elements (including relative positions like {@code -1}, will be ignored during the
1249   * append. For example, the second {@code 0} position is lost in the result.
1250   * <p>
1251   * {@code assert new Fields( 0, "a" ).append( new Fields( 0, "b" ).equals( new Fields( 0, "a", 2, "b" )}
1252   * <p>
1253   * See {@link #subtract(Fields)} for removing field names.
1254   *
1255   * @param fields of type Fields
1256   * @return Fields
1257   */
1258  public Fields append( Fields fields )
1259    {
1260    return appendInternal( fields, false );
1261    }
1262
1263  /**
1264   * Method is used for appending the given Fields instance to this instance, into a new Fields instance
1265   * suitable for use as a field selector.
1266   * <p>
1267   * That is, any positional elements will be retained during the append. For example, the {@code 5} and {@code 0}
1268   * are retained in the result.
1269   * <p>
1270   * {@code assert new Fields( 5, "a" ).append( new Fields( 0, "b" ).equals( new Fields( 5, "a", 0, "b" )}
1271   * <p>
1272   * Note any relative positional elements are retained, thus appending two Fields each declaring {@code -1}
1273   * position will result in a TupleException noting duplicate fields.
1274   * <p>
1275   * See {@link #subtract(Fields)} for removing field names.
1276   *
1277   * @param fields of type Fields
1278   * @return Fields
1279   */
1280  public Fields appendSelector( Fields fields )
1281    {
1282    return appendInternal( fields, true );
1283    }
1284
1285  private Fields appendInternal( Fields fields, boolean isSelect )
1286    {
1287    if( fields == null )
1288      return this;
1289
1290    // allow unordered fields to be appended to build more complex selectors
1291    if( this.isAll() || fields.isAll() )
1292      throw new TupleException( "cannot append fields: " + this.print() + " + " + fields.print() );
1293
1294    if( ( this.isUnknown() || this.size() == 0 ) && fields.isUnknown() )
1295      return UNKNOWN;
1296
1297    if( fields.isNone() )
1298      return this;
1299
1300    if( this.isNone() )
1301      return fields;
1302
1303    Set<Comparable> names = new HashSet<Comparable>();
1304
1305    // init the Field
1306    Fields result = size( this.size() + fields.size() );
1307
1308    // copy over field names from this side
1309    copyRetain( names, result, this, 0, isSelect );
1310    // copy over field names from that side
1311    copyRetain( names, result, fields, this.size(), isSelect );
1312
1313    if( this.isUnknown() || fields.isUnknown() )
1314      result.kind = Kind.UNKNOWN;
1315
1316    if( ( this.isNone() || this.types != null ) && fields.types != null )
1317      {
1318      result.types = new Type[ this.size() + fields.size() ];
1319
1320      if( this.types != null ) // supports appending to NONE
1321        System.arraycopy( this.types, 0, result.types, 0, this.size() );
1322
1323      System.arraycopy( fields.types, 0, result.types, this.size(), fields.size() );
1324      }
1325
1326    return result;
1327    }
1328
1329  /**
1330   * Method rename will rename the from fields to the values in to to fields. Fields may contain field names, or
1331   * positions.
1332   * <p>
1333   * Using positions is useful to remove a field name put keep its place in the Tuple stream.
1334   *
1335   * @param from of type Fields
1336   * @param to   of type Fields
1337   * @return Fields
1338   */
1339  public Fields rename( Fields from, Fields to )
1340    {
1341    if( this.isSubstitution() || this.isUnknown() )
1342      throw new TupleException( "cannot rename fields in a substitution or unknown Fields instance: " + this.print() );
1343
1344    if( from.size() != to.size() )
1345      throw new TupleException( "from and to fields must be the same size" );
1346
1347    if( from.isSubstitution() || from.isUnknown() )
1348      throw new TupleException( "from fields may not be a substitution or unknown" );
1349
1350    if( to.isSubstitution() || to.isUnknown() )
1351      throw new TupleException( "to fields may not be a substitution or unknown" );
1352
1353    Comparable[] newFields = Arrays.copyOf( this.fields, this.fields.length );
1354
1355    int[] pos = getPos( from );
1356
1357    for( int i = 0; i < pos.length; i++ )
1358      newFields[ pos[ i ] ] = to.fields[ i ];
1359
1360    Type[] newTypes = null;
1361
1362    if( this.types != null && to.types != null )
1363      {
1364      newTypes = copyTypes( this.types, this.size() );
1365
1366      for( int i = 0; i < pos.length; i++ )
1367        newTypes[ pos[ i ] ] = to.types[ i ];
1368      }
1369
1370    return new Fields( newFields, newTypes );
1371    }
1372
1373  /**
1374   * Method project will return a new Fields instance similar to the given fields instance
1375   * except any absolute positional elements will be replaced by the current field names, if any.
1376   *
1377   * @param fields of type Fields
1378   * @return Fields
1379   */
1380  public Fields project( Fields fields )
1381    {
1382    if( fields == null )
1383      return this;
1384
1385    Fields results = size( fields.size() ).applyTypes( fields.getTypes() );
1386
1387    for( int i = 0; i < fields.fields.length; i++ )
1388      {
1389      if( fields.fields[ i ] instanceof String )
1390        results.fields[ i ] = fields.fields[ i ];
1391      else if( this.fields[ i ] instanceof String )
1392        results.fields[ i ] = this.fields[ i ];
1393      else
1394        results.fields[ i ] = i;
1395      }
1396
1397    return results;
1398    }
1399
1400  private static void copy( Set<String> names, Fields result, Fields fields, int offset )
1401    {
1402    for( int i = 0; i < fields.size(); i++ )
1403      {
1404      Comparable field = fields.get( i );
1405
1406      if( !( field instanceof String ) )
1407        continue;
1408
1409      if( names != null )
1410        {
1411        if( names.contains( field ) )
1412          throw new TupleException( "field name already exists: " + field );
1413
1414        names.add( (String) field );
1415        }
1416
1417      result.set( i + offset, field );
1418      }
1419    }
1420
1421  /**
1422   * Retains any relative positional elements like -1, but checks for duplicates
1423   *
1424   * @param names
1425   * @param result
1426   * @param fields
1427   * @param offset
1428   * @param isSelect
1429   */
1430  private static void copyRetain( Set<Comparable> names, Fields result, Fields fields, int offset, boolean isSelect )
1431    {
1432    for( int i = 0; i < fields.size(); i++ )
1433      {
1434      Comparable field = fields.get( i );
1435
1436      if( !isSelect && field instanceof Integer )
1437        continue;
1438
1439      if( names != null )
1440        {
1441        if( names.contains( field ) )
1442          throw new TupleException( "field name already exists: " + field );
1443
1444        names.add( field );
1445        }
1446
1447      result.set( i + offset, field );
1448      }
1449    }
1450
1451  /**
1452   * Method verifyContains tests if this instance contains the field names and positions specified in the given
1453   * fields instance. If the test fails, a {@link TupleException} is thrown.
1454   *
1455   * @param fields of type Fields
1456   * @throws TupleException when one or more fields are not contained in this instance.
1457   */
1458  public void verifyContains( Fields fields )
1459    {
1460    if( isUnknown() )
1461      return;
1462
1463    try
1464      {
1465      getPos( fields );
1466      }
1467    catch( TupleException exception )
1468      {
1469      throw new TupleException( "these fields " + print() + ", do not contain " + fields.print() );
1470      }
1471    }
1472
1473  /**
1474   * Method contains returns true if this instance contains the field names and positions specified in the given
1475   * fields instance.
1476   *
1477   * @param fields of type Fields
1478   * @return boolean
1479   */
1480  public boolean contains( Fields fields )
1481    {
1482    try
1483      {
1484      getPos( fields );
1485      return true;
1486      }
1487    catch( Exception exception )
1488      {
1489      return false;
1490      }
1491    }
1492
1493  /**
1494   * Method compareTo compares this instance to the given Fields instance.
1495   *
1496   * @param other of type Fields
1497   * @return int
1498   */
1499  public int compareTo( Fields other )
1500    {
1501    if( other.size() != size() )
1502      return other.size() < size() ? 1 : -1;
1503
1504    for( int i = 0; i < size(); i++ )
1505      {
1506      int c = get( i ).compareTo( other.get( i ) );
1507
1508      if( c != 0 )
1509        return c;
1510      }
1511
1512    return 0;
1513    }
1514
1515  /**
1516   * Method compareTo implements {@link Comparable#compareTo(Object)}.
1517   *
1518   * @param other of type Object
1519   * @return int
1520   */
1521  public int compareTo( Object other )
1522    {
1523    if( other instanceof Fields )
1524      return compareTo( (Fields) other );
1525    else
1526      return -1;
1527    }
1528
1529  /**
1530   * Method print returns a String representation of this instance.
1531   *
1532   * @return String
1533   */
1534  public String print()
1535    {
1536    return "[" + toString() + "]";
1537    }
1538
1539  /**
1540   * Method printLong returns a String representation of this instance along with the size.
1541   *
1542   * @return String
1543   */
1544  public String printVerbose()
1545    {
1546    String fieldsString = toString();
1547
1548    return "[{" + ( isDefined() ? size() : "?" ) + "}:" + fieldsString + "]";
1549    }
1550
1551  @Override
1552  public String toString()
1553    {
1554    String string;
1555
1556    if( isOrdered() )
1557      string = orderedToString();
1558    else
1559      string = unorderedToString();
1560
1561    if( types != null )
1562      string += " | " + Util.join( Util.simpleTypeNames( types ), ", " );
1563
1564    return string;
1565    }
1566
1567  private String orderedToString()
1568    {
1569    StringBuffer buffer = new StringBuffer();
1570
1571    if( size() != 0 )
1572      {
1573      int startIndex = get( 0 ) instanceof Number ? (Integer) get( 0 ) : 0;
1574
1575      for( int i = 0; i < size(); i++ )
1576        {
1577        Comparable field = get( i );
1578
1579        if( field instanceof Number )
1580          {
1581          if( i + 1 == size() || !( get( i + 1 ) instanceof Number ) )
1582            {
1583            if( buffer.length() != 0 )
1584              buffer.append( ", " );
1585
1586            if( startIndex != i )
1587              buffer.append( startIndex ).append( ":" ).append( field );
1588            else
1589              buffer.append( i );
1590
1591            startIndex = i;
1592            }
1593
1594          continue;
1595          }
1596
1597        if( i != 0 )
1598          buffer.append( ", " );
1599
1600        if( field instanceof String )
1601          buffer.append( "\'" ).append( field ).append( "\'" );
1602        else if( field instanceof Fields )
1603          buffer.append( ( (Fields) field ).print() );
1604
1605        startIndex = i + 1;
1606        }
1607      }
1608
1609    if( kind != null )
1610      {
1611      if( buffer.length() != 0 )
1612        buffer.append( ", " );
1613      buffer.append( kind );
1614      }
1615
1616    return buffer.toString();
1617    }
1618
1619  private String unorderedToString()
1620    {
1621    StringBuffer buffer = new StringBuffer();
1622
1623    for( Object field : get() )
1624      {
1625      if( buffer.length() != 0 )
1626        buffer.append( ", " );
1627
1628      if( field instanceof String )
1629        buffer.append( "\'" ).append( field ).append( "\'" );
1630      else if( field instanceof Fields )
1631        buffer.append( ( (Fields) field ).print() );
1632      else
1633        buffer.append( field );
1634      }
1635
1636    if( kind != null )
1637      {
1638      if( buffer.length() != 0 )
1639        buffer.append( ", " );
1640      buffer.append( kind );
1641      }
1642
1643    return buffer.toString();
1644    }
1645
1646  /**
1647   * Method size returns the number of field positions in this instance.
1648   *
1649   * @return int
1650   */
1651  public final int size()
1652    {
1653    return fields.length;
1654    }
1655
1656  /**
1657   * Method applyFields returns a new Fields instance with the given field names, replacing any existing type
1658   * information within the new instance.
1659   * <p>
1660   * The Comparable array must be the same length as the number for fields in this instance.
1661   *
1662   * @param fields the field names of this Fields object.
1663   * @return returns a new instance of Fields with this instances types and the given field names
1664   */
1665  public Fields applyFields( Comparable... fields )
1666    {
1667    Fields result = new Fields( fields );
1668
1669    if( types == null )
1670      return result;
1671
1672    if( types.length != result.size() )
1673      throw new IllegalArgumentException( "given number of field names must match current fields size" );
1674
1675    result.types = copyTypes( types, types.length ); // make copy as Class[] could be passed in
1676
1677    return result;
1678    }
1679
1680  /**
1681   * Method applyType should be used to associate a {@link java.lang.reflect.Type} with a given field name or position.
1682   * A new instance of Fields will be returned, this instance will not be modified.
1683   * <p>
1684   * {@code fieldName} may optionally be a {@link Fields} instance. Only the first field name or position will
1685   * be considered.
1686   *
1687   * @param fieldName of type Comparable
1688   * @param type      of type Type
1689   */
1690  public Fields applyType( Comparable fieldName, Type type )
1691    {
1692    if( type == null )
1693      throw new IllegalArgumentException( "given type must not be null" );
1694
1695    int pos;
1696
1697    try
1698      {
1699      pos = getPos( asFieldName( fieldName ) );
1700      }
1701    catch( FieldsResolverException exception )
1702      {
1703      throw new IllegalArgumentException( "given field name was not found: " + fieldName, exception );
1704      }
1705
1706    Fields results = new Fields( fields );
1707
1708    results.types = this.types == null ? new Type[ size() ] : copyTypes( this.types, this.types.length );
1709    results.types[ pos ] = type;
1710
1711    return results;
1712    }
1713
1714  /**
1715   * Method applyType should be used to associate a {@link java.lang.reflect.Type} with all positions in the current instance.
1716   * A new instance of Fields will be returned, this instance will not be modified.
1717   *
1718   * @param type of type Type
1719   */
1720  public Fields applyTypeToAll( Type type )
1721    {
1722    Fields result = new Fields( fields );
1723
1724    if( type == null ) // allows for type erasure
1725      return result;
1726
1727    Type[] copy = new Type[ result.size() ];
1728
1729    Arrays.fill( copy, type );
1730
1731    result.types = copy;
1732
1733    return result;
1734    }
1735
1736  /**
1737   * Method applyType should be used to associate {@link java.lang.reflect.Type} with a given field name or position
1738   * as declared in the given Fields parameter.
1739   * <p>
1740   * A new instance of Fields will be returned, this instance will not be modified.
1741   * <p>
1742   *
1743   * @param fields of type Fields
1744   */
1745  public Fields applyTypes( Fields fields )
1746    {
1747    Fields result = new Fields( this.fields, this.types );
1748
1749    for( Comparable field : fields )
1750      result = result.applyType( field, fields.getType( fields.getPos( field ) ) );
1751
1752    return result;
1753    }
1754
1755  /**
1756   * Method applyTypes returns a new Fields instance with the given types, replacing any existing type
1757   * information within the new instance.
1758   * <p>
1759   * The Class array must be the same length as the number for fields in this instance.
1760   * <p>
1761   * If no values are given, the resulting Fields instance will have no type information.
1762   *
1763   * @param types the class types of this Fields object.
1764   * @return returns a new instance of Fields with this instances field names and the given types, if any
1765   */
1766  public Fields applyTypes( Type... types )
1767    {
1768    Fields result = new Fields( fields );
1769
1770    if( types == null || types.length == 0 ) // allows for type erasure
1771      return result;
1772
1773    if( types.length != size() )
1774      throw new IllegalArgumentException( "given number of class instances must match fields size" );
1775
1776    for( Type type : types )
1777      {
1778      if( type == null )
1779        throw new IllegalArgumentException( "type must not be null" );
1780      }
1781
1782    result.types = copyTypes( types, types.length ); // make copy as Class[] could be passed in
1783
1784    return result;
1785    }
1786
1787  /**
1788   * Method unApplyTypes returns a new Fields instance without any type information.
1789   *
1790   * @return returns a new instance of Fields with this instances field names and no type information
1791   */
1792  public Fields unApplyTypes()
1793    {
1794    return applyTypes();
1795    }
1796
1797  /**
1798   * Returns the Type at the given position or having the fieldName.
1799   *
1800   * @param fieldName of type String or Number
1801   * @return the Type
1802   */
1803  public Type getType( Comparable fieldName )
1804    {
1805    if( !hasTypes() )
1806      return null;
1807
1808    return getType( getPos( fieldName ) );
1809    }
1810
1811  public Type getType( int pos )
1812    {
1813    if( !hasTypes() )
1814      return null;
1815
1816    return this.types[ pos ];
1817    }
1818
1819  /**
1820   * Returns the Class for the given position value.
1821   * <p>
1822   * If the underlying value is of type {@link CoercibleType}, the result of
1823   * {@link cascading.tuple.type.CoercibleType#getCanonicalType()} will returned.
1824   *
1825   * @param fieldName of type String or Number
1826   * @return type Class
1827   */
1828  public Class getTypeClass( Comparable fieldName )
1829    {
1830    if( !hasTypes() )
1831      return null;
1832
1833    return getTypeClass( getPos( fieldName ) );
1834    }
1835
1836  public Class getTypeClass( int pos )
1837    {
1838    Type type = getType( pos );
1839
1840    if( type instanceof CoercibleType )
1841      return ( (CoercibleType) type ).getCanonicalType();
1842
1843    return (Class) type;
1844    }
1845
1846  protected void setType( int pos, Type type )
1847    {
1848    if( type == null )
1849      throw new IllegalArgumentException( "type may not be null" );
1850
1851    this.types[ pos ] = type;
1852    }
1853
1854  /**
1855   * Returns a copy of the current types Type[] if any, else null.
1856   *
1857   * @return of type Type[]
1858   */
1859  public Type[] getTypes()
1860    {
1861    return copyTypes( types, size() );
1862    }
1863
1864  /**
1865   * Returns a copy of the current types Class[] if any, else null.
1866   * <p>
1867   * If any underlying value is of type {@link CoercibleType}, the result of
1868   * {@link cascading.tuple.type.CoercibleType#getCanonicalType()} will returned.
1869   *
1870   * @return of type Class
1871   */
1872  public Class[] getTypesClasses()
1873    {
1874    if( types == null )
1875      return null;
1876
1877    Class[] classes = new Class[ types.length ];
1878
1879    for( int i = 0; i < types.length; i++ )
1880      {
1881      if( types[ i ] instanceof CoercibleType )
1882        classes[ i ] = ( (CoercibleType) types[ i ] ).getCanonicalType();
1883      else
1884        classes[ i ] = (Class) types[ i ]; // this throws a more helpful exception vs arraycopy
1885      }
1886
1887    return classes;
1888    }
1889
1890  private static Type[] copyTypes( Type[] types, int size )
1891    {
1892    if( types == null )
1893      return null;
1894
1895    Type[] copy = new Type[ size ];
1896
1897    if( types.length != size )
1898      throw new IllegalArgumentException( "types array must be same size as fields array" );
1899
1900    System.arraycopy( types, 0, copy, 0, size );
1901
1902    return copy;
1903    }
1904
1905  /**
1906   * Returns true if there are types associated with this instance.
1907   *
1908   * @return boolean
1909   */
1910  public final boolean hasTypes()
1911    {
1912    return types != null;
1913    }
1914
1915  /**
1916   * Method setComparator should be used to associate a {@link java.util.Comparator} with a given field name or position.
1917   * <p>
1918   * {@code fieldName} may optionally be a {@link Fields} instance. Only the first field name or position will
1919   * be considered.
1920   *
1921   * @param fieldName  of type Comparable
1922   * @param comparator of type Comparator
1923   */
1924  public void setComparator( Comparable fieldName, Comparator comparator )
1925    {
1926    if( !( comparator instanceof Serializable ) )
1927      throw new IllegalArgumentException( "given comparator must be serializable" );
1928
1929    if( comparators == null )
1930      comparators = new Comparator[ size() ];
1931
1932    try
1933      {
1934      comparators[ getPos( asFieldName( fieldName ) ) ] = comparator;
1935      }
1936    catch( FieldsResolverException exception )
1937      {
1938      throw new IllegalArgumentException( "given field name was not found: " + fieldName, exception );
1939      }
1940    }
1941
1942  /**
1943   * Method setComparators sets all the comparators of this Fields object. The Comparator array
1944   * must be the same length as the number for fields in this instance.
1945   *
1946   * @param comparators the comparators of this Fields object.
1947   */
1948  public void setComparators( Comparator... comparators )
1949    {
1950    if( comparators.length != size() )
1951      throw new IllegalArgumentException( "given number of comparator instances must match fields size" );
1952
1953    for( Comparator comparator : comparators )
1954      {
1955      if( !( comparator instanceof Serializable ) )
1956        throw new IllegalArgumentException( "comparators must be serializable" );
1957      }
1958
1959    this.comparators = comparators;
1960    }
1961
1962  protected static Comparable asFieldName( Comparable fieldName )
1963    {
1964    if( fieldName instanceof Fields )
1965      {
1966      Fields fields = (Fields) fieldName;
1967
1968      if( !fields.isDefined() )
1969        throw new TupleException( "given Fields instance must explicitly declare one field name or position: " + fields.printVerbose() );
1970
1971      fieldName = fields.get( 0 );
1972      }
1973
1974    return fieldName;
1975    }
1976
1977  protected Comparator getComparator( Comparable fieldName )
1978    {
1979    if( comparators == null )
1980      return null;
1981
1982    try
1983      {
1984      return comparators[ getPos( asFieldName( fieldName ) ) ];
1985      }
1986    catch( FieldsResolverException exception )
1987      {
1988      return null;
1989      }
1990    }
1991
1992  /**
1993   * Method getComparators returns the comparators of this Fields object.
1994   *
1995   * @return the comparators (type Comparator[]) of this Fields object.
1996   */
1997  public Comparator[] getComparators()
1998    {
1999    Comparator[] copy = new Comparator[ size() ];
2000
2001    if( comparators != null )
2002      System.arraycopy( comparators, 0, copy, 0, size() );
2003
2004    return copy;
2005    }
2006
2007  /**
2008   * Method hasComparators test if this Fields instance has Comparators.
2009   *
2010   * @return boolean
2011   */
2012  public boolean hasComparators()
2013    {
2014    return comparators != null;
2015    }
2016
2017  @Override
2018  public int compare( Tuple lhs, Tuple rhs )
2019    {
2020    return lhs.compareTo( comparators, rhs );
2021    }
2022
2023  @Override
2024  public boolean equals( Object object )
2025    {
2026    if( this == object )
2027      return true;
2028    if( object == null || getClass() != object.getClass() )
2029      return false;
2030
2031    Fields fields = (Fields) object;
2032
2033    return equalsFields( fields ) && Arrays.equals( types, fields.types );
2034    }
2035
2036  /**
2037   * Method equalsFields compares only the internal field names and postions only between this and the given Fields
2038   * instance. Type information is ignored.
2039   *
2040   * @param fields of type int
2041   * @return true if this and the given instance have the same positions and/or field names.
2042   */
2043  public boolean equalsFields( Fields fields )
2044    {
2045    return fields != null && this.kind == fields.kind && Arrays.equals( get(), fields.get() );
2046    }
2047
2048  @Override
2049  public int hashCode()
2050    {
2051    if( hashCode == 0 )
2052      hashCode = get() != null ? Arrays.hashCode( get() ) : 0;
2053
2054    return hashCode;
2055    }
2056  }