001/*
002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
004 *
005 * Project and contact information: http://www.cascading.org/
006 *
007 * This file is part of the Cascading project.
008 *
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *     http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 */
021
022package cascading.flow;
023
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Collection;
027import java.util.HashMap;
028import java.util.LinkedHashMap;
029import java.util.List;
030import java.util.Map;
031
032import cascading.operation.AssertionLevel;
033import cascading.operation.DebugLevel;
034import cascading.pipe.Checkpoint;
035import cascading.pipe.Pipe;
036import cascading.property.UnitOfWorkDef;
037import cascading.tap.Tap;
038import cascading.util.Util;
039
040/**
041 * Class FlowDef is a fluent interface for defining a {@link Flow}.
042 * <p>
043 * This allows for ad-hoc building of Flow data and meta-data, like tags.
044 * <p>
045 * Instead of calling one of the {@link FlowConnector} connect methods, {@link FlowConnector#connect(FlowDef)}
046 * can be called.
047 */
048public class FlowDef extends UnitOfWorkDef<FlowDef>
049  {
050  protected Map<String, Tap> sources = new HashMap<String, Tap>();
051  protected Map<String, Tap> sinks = new HashMap<String, Tap>();
052  protected Map<String, Tap> traps = new HashMap<String, Tap>();
053  protected Map<String, Tap> checkpoints = new HashMap<String, Tap>();
054
055  protected List<String> classPath = new ArrayList<String>();
056  protected List<Pipe> tails = new ArrayList<Pipe>();
057  protected List<AssemblyPlanner> assemblyPlanners = new ArrayList<AssemblyPlanner>();
058
059  protected HashMap<String, String> flowDescriptor = new LinkedHashMap<String, String>();
060
061  protected AssertionLevel assertionLevel;
062  protected DebugLevel debugLevel;
063
064  protected String runID;
065
066  /**
067   * Creates a new instance of a FlowDef.
068   *
069   * @return a FlowDef
070   */
071  public static FlowDef flowDef()
072    {
073    return new FlowDef();
074    }
075
076  /** Constructor FlowDef creates a new FlowDef instance. */
077  public FlowDef()
078    {
079    }
080
081  protected FlowDef( FlowDef flowDef, Map<String, Tap> sources, Map<String, Tap> sinks, Map<String, Tap> traps, Map<String, Tap> checkpoints )
082    {
083    super( flowDef );
084
085    this.sources = sources;
086    this.sinks = sinks;
087    this.traps = traps;
088    this.checkpoints = checkpoints;
089
090    this.classPath = flowDef.classPath;
091    this.tails = flowDef.tails;
092    this.assemblyPlanners = flowDef.assemblyPlanners;
093    this.flowDescriptor = flowDef.flowDescriptor;
094    this.assertionLevel = flowDef.assertionLevel;
095    this.debugLevel = flowDef.debugLevel;
096    this.runID = flowDef.runID;
097    }
098
099  /**
100   * Method getAssemblyPlanners returns the current registered AssemblyPlanners.
101   *
102   * @return a List of AssemblyPlanner instances
103   */
104  public List<AssemblyPlanner> getAssemblyPlanners()
105    {
106    return assemblyPlanners;
107    }
108
109  /**
110   * Method addAssemblyPlanner adds new AssemblyPlanner instances to be evaluated.
111   *
112   * @param assemblyPlanner of type AssemblyPlanner
113   * @return a FlowDef
114   */
115  public FlowDef addAssemblyPlanner( AssemblyPlanner assemblyPlanner )
116    {
117    assemblyPlanners.add( assemblyPlanner );
118    addDescriptions( assemblyPlanner.getFlowDescriptor() );
119
120    return this;
121    }
122
123  /**
124   * Method getSources returns the sources of this FlowDef object.
125   *
126   * @return the sources (type Map) of this FlowDef object.
127   */
128  public Map<String, Tap> getSources()
129    {
130    return sources;
131    }
132
133  /**
134   * Method getSourcesCopy returns a copy of the sources Map.
135   *
136   * @return the sourcesCopy (type Map) of this FlowDef object.
137   */
138  public Map<String, Tap> getSourcesCopy()
139    {
140    return new HashMap<String, Tap>( sources );
141    }
142
143  /**
144   * Method getFlowDescriptor returns the  flowDescriptor of this FlowDef.
145   *
146   * @return the flowDescriptor of this FlowDef object.
147   */
148  public HashMap<String, String> getFlowDescriptor()
149    {
150    return flowDescriptor;
151    }
152
153  /**
154   * Method addSource adds a new named source {@link Tap} for use in the resulting {@link Flow}.
155   *
156   * @param name   of String
157   * @param source of Tap
158   * @return FlowDef
159   */
160  public FlowDef addSource( String name, Tap source )
161    {
162    if( sources.containsKey( name ) )
163      throw new IllegalArgumentException( "cannot add duplicate source: " + name );
164
165    sources.put( name, source );
166    return this;
167    }
168
169  /**
170   * Method addSource adds a new source {@link Tap} named after the given {@link Pipe} for use in the resulting {@link Flow}.
171   * <p>
172   * If the given pipe is not a head pipe, it will be resolved. If more than one is found, an
173   * {@link IllegalArgumentException} will be thrown.
174   *
175   * @param pipe   of Pipe
176   * @param source of Tap
177   * @return FlowDef
178   */
179  public FlowDef addSource( Pipe pipe, Tap source )
180    {
181    if( pipe == null )
182      throw new IllegalArgumentException( "pipe may not be null" );
183
184    Pipe[] heads = pipe.getHeads();
185
186    if( heads.length != 1 )
187      throw new IllegalArgumentException( "pipe has too many heads, found: " + Arrays.toString( Pipe.names( heads ) ) );
188
189    addSource( heads[ 0 ].getName(), source );
190    return this;
191    }
192
193  /**
194   * Method addSources adds a map of name and {@link Tap} pairs.
195   *
196   * @param sources of Map
197   * @return FlowDef
198   */
199  public FlowDef addSources( Map<String, Tap> sources )
200    {
201    if( sources != null )
202      {
203      for( Map.Entry<String, Tap> entry : sources.entrySet() )
204        addSource( entry.getKey(), entry.getValue() );
205      }
206
207    return this;
208    }
209
210  /**
211   * Method addDescription adds a user readable description to the flowDescriptor.
212   * <p>
213   * This uses the {@link FlowDescriptors#DESCRIPTION} key.
214   */
215  public FlowDef addDescription( String description )
216    {
217    addDescription( FlowDescriptors.DESCRIPTION, description );
218
219    return this;
220    }
221
222  /**
223   * Method addDescription adds a description to the flowDescriptor.
224   * <p>
225   * Flow descriptions provide meta-data to monitoring systems describing the workload a given Flow represents.
226   * For known description types, see {@link FlowDescriptors}.
227   * <p>
228   * If an existing key exists, it will be appended to the original value using
229   * {@link FlowDescriptors#VALUE_SEPARATOR}.
230   *
231   * @param key   The key as a String.
232   * @param value The value as a String.
233   * @return FlowDef
234   */
235  public FlowDef addDescription( String key, String value )
236    {
237    if( Util.isEmpty( value ) ) // do nothing
238      return this;
239
240    if( flowDescriptor.containsKey( key ) )
241      {
242      String original = flowDescriptor.get( key );
243
244      if( !Util.isEmpty( original ) )
245        value = original + FlowDescriptors.VALUE_SEPARATOR + value;
246      }
247
248    flowDescriptor.put( key, value );
249
250    return this;
251    }
252
253  /**
254   * Method addProperties adds all properties in the given map in order to the flowDescriptor. If the given Map has
255   * an explicit order, it will be preserved.
256   * <p>
257   * Flow descriptions provide meta-data to monitoring systems describing the workload a given Flow represents.
258   * For known description types, see {@link FlowDescriptors}.
259   *
260   * @param descriptions The properties to be added to the map.
261   * @return FlowDef
262   */
263  public FlowDef addDescriptions( Map<String, String> descriptions )
264    {
265    for( Map.Entry<String, String> entry : descriptions.entrySet() )
266      addDescription( entry.getKey(), entry.getValue() );
267
268    return this;
269    }
270
271  /**
272   * Method getSinks returns the sinks of this FlowDef object.
273   *
274   * @return the sinks (type Map) of this FlowDef object.
275   */
276  public Map<String, Tap> getSinks()
277    {
278    return sinks;
279    }
280
281  /**
282   * Method getSinksCopy returns a copy of the sink Map.
283   *
284   * @return the sinksCopy (type Map) of this FlowDef object.
285   */
286  public Map<String, Tap> getSinksCopy()
287    {
288    return new HashMap<String, Tap>( sinks );
289    }
290
291  /**
292   * Method addSink adds a new named sink {@link Tap} for use in the resulting {@link Flow}.
293   *
294   * @param name of String
295   * @param sink of Tap
296   * @return FlowDef
297   */
298  public FlowDef addSink( String name, Tap sink )
299    {
300    if( sinks.containsKey( name ) )
301      throw new IllegalArgumentException( "cannot add duplicate sink: " + name );
302
303    sinks.put( name, sink );
304    return this;
305    }
306
307  /**
308   * Method addSink adds a new sink {@link Tap} named after the given {@link Pipe} for use in the resulting {@link Flow}.
309   *
310   * @param tail of Pipe
311   * @param sink of Tap
312   * @return FlowDef
313   */
314  public FlowDef addSink( Pipe tail, Tap sink )
315    {
316    addSink( tail.getName(), sink );
317    return this;
318    }
319
320  /**
321   * Method addTailSink adds the tail {@link Pipe} and sink {@link Tap} to this FlowDef.
322   * <p>
323   * This is a convenience method for adding both a tail and sink simultaneously. There isn't a similar method
324   * for heads and sources as the head Pipe can always be derived.
325   *
326   * @param tail of Pipe
327   * @param sink of Tap
328   * @return FlowDef
329   */
330  public FlowDef addTailSink( Pipe tail, Tap sink )
331    {
332    addSink( tail.getName(), sink );
333    addTail( tail );
334    return this;
335    }
336
337  /**
338   * Method addSinks adds a Map of the named and {@link Tap} pairs.
339   *
340   * @param sinks of Map
341   * @return FlowDef
342   */
343  public FlowDef addSinks( Map<String, Tap> sinks )
344    {
345    if( sinks != null )
346      {
347      for( Map.Entry<String, Tap> entry : sinks.entrySet() )
348        addSink( entry.getKey(), entry.getValue() );
349      }
350
351    return this;
352    }
353
354  /**
355   * Method getTraps returns the traps of this FlowDef object.
356   *
357   * @return the traps (type Map) of this FlowDef object.
358   */
359  public Map<String, Tap> getTraps()
360    {
361    return traps;
362    }
363
364  /**
365   * Method getTrapsCopy returns a copy of the trap Map.
366   *
367   * @return the trapsCopy (type Map) of this FlowDef object.
368   */
369  public Map<String, Tap> getTrapsCopy()
370    {
371    return new HashMap<String, Tap>( traps );
372    }
373
374  /**
375   * Method addTrap adds a new named trap {@link Tap} for use in the resulting {@link Flow}.
376   *
377   * @param name of String
378   * @param trap of Tap
379   * @return FlowDef
380   */
381  public FlowDef addTrap( String name, Tap trap )
382    {
383    if( traps.containsKey( name ) )
384      throw new IllegalArgumentException( "cannot add duplicate trap: " + name );
385
386    traps.put( name, trap );
387    return this;
388    }
389
390  /**
391   * Method addTrap adds a new trap {@link Tap} named after the given {@link Pipe} for use in the resulting {@link Flow}.
392   *
393   * @param pipe of Pipe
394   * @param trap of Tap
395   * @return FlowDef
396   */
397  public FlowDef addTrap( Pipe pipe, Tap trap )
398    {
399    addTrap( pipe.getName(), trap );
400    return this;
401    }
402
403  /**
404   * Method addTraps adds a Map of the names and {@link Tap} pairs.
405   *
406   * @param traps of Map
407   * @return FlowDef
408   */
409  public FlowDef addTraps( Map<String, Tap> traps )
410    {
411    if( traps != null )
412      {
413      for( Map.Entry<String, Tap> entry : traps.entrySet() )
414        addTrap( entry.getKey(), entry.getValue() );
415      }
416
417    return this;
418    }
419
420  /**
421   * Method getCheckpoints returns the checkpoint taps of this FlowDef object.
422   *
423   * @return the checkpoints (type Map) of this FlowDef object.
424   */
425  public Map<String, Tap> getCheckpoints()
426    {
427    return checkpoints;
428    }
429
430  /**
431   * Method getCheckpointsCopy returns a copy of the checkpoint tap Map.
432   *
433   * @return the checkpointsCopy (type Map) of this FlowDef object.
434   */
435  public Map<String, Tap> getCheckpointsCopy()
436    {
437    return new HashMap<String, Tap>( checkpoints );
438    }
439
440  /**
441   * Method addCheckpoint adds a new named checkpoint {@link Tap} for use in the resulting {@link Flow}.
442   *
443   * @param name       of String
444   * @param checkpoint of Tap
445   * @return FlowDef
446   */
447  public FlowDef addCheckpoint( String name, Tap checkpoint )
448    {
449    if( checkpoints.containsKey( name ) )
450      throw new IllegalArgumentException( "cannot add duplicate checkpoint: " + name );
451
452    checkpoints.put( name, checkpoint );
453    return this;
454    }
455
456  /**
457   * Method addCheckpoint adds a new checkpoint {@link Tap} named after the given {@link Checkpoint} for use in the resulting {@link Flow}.
458   *
459   * @param pipe       of Pipe
460   * @param checkpoint of Tap
461   * @return FlowDef
462   */
463  public FlowDef addCheckpoint( Checkpoint pipe, Tap checkpoint )
464    {
465    addCheckpoint( pipe.getName(), checkpoint );
466    return this;
467    }
468
469  /**
470   * Method addCheckpoints adds a Map of the names and {@link Tap} pairs.
471   *
472   * @param checkpoints of Map
473   * @return FlowDef
474   */
475  public FlowDef addCheckpoints( Map<String, Tap> checkpoints )
476    {
477    if( checkpoints != null )
478      {
479      for( Map.Entry<String, Tap> entry : checkpoints.entrySet() )
480        addCheckpoint( entry.getKey(), entry.getValue() );
481      }
482
483    return this;
484    }
485
486  /**
487   * Method getTails returns all the current pipe assembly tails the FlowDef holds.
488   *
489   * @return the tails (type List) of this FlowDef object.
490   */
491  public List<Pipe> getTails()
492    {
493    return tails;
494    }
495
496  /**
497   * Method getTailsArray returns all the current pipe assembly tails the FlowDef holds.
498   *
499   * @return the tailsArray (type Pipe[]) of this FlowDef object.
500   */
501  public Pipe[] getTailsArray()
502    {
503    return tails.toArray( new Pipe[ tails.size() ] );
504    }
505
506  /**
507   * Method addTail adds a new {@link Pipe} to this FlowDef that represents a tail in a pipe assembly.
508   * <p>
509   * Be sure to add a sink tap that has the same name as this tail.
510   *
511   * @param tail of Pipe
512   * @return FlowDef
513   */
514  public FlowDef addTail( Pipe tail )
515    {
516    if( tail != null )
517      this.tails.add( tail );
518
519    return this;
520    }
521
522  /**
523   * Method addTails adds a Collection of tails.
524   *
525   * @param tails of Collection
526   * @return FlowDef
527   */
528  public FlowDef addTails( Collection<Pipe> tails )
529    {
530    for( Pipe tail : tails )
531      addTail( tail );
532
533    return this;
534    }
535
536  /**
537   * Method addTails adds an array of tails.
538   *
539   * @param tails of Pipe...
540   * @return FlowDef
541   */
542  public FlowDef addTails( Pipe... tails )
543    {
544    for( Pipe tail : tails )
545      addTail( tail );
546
547    return this;
548    }
549
550  public FlowDef setAssertionLevel( AssertionLevel assertionLevel )
551    {
552    this.assertionLevel = assertionLevel;
553
554    return this;
555    }
556
557  public AssertionLevel getAssertionLevel()
558    {
559    return assertionLevel;
560    }
561
562  public FlowDef setDebugLevel( DebugLevel debugLevel )
563    {
564    this.debugLevel = debugLevel;
565
566    return this;
567    }
568
569  public DebugLevel getDebugLevel()
570    {
571    return debugLevel;
572    }
573
574  /**
575   * Method setRunID sets the checkpoint run or execution ID to be used to find prior failed runs against
576   * this runID.
577   * <p>
578   * When given, and a {@link Flow} fails to execute, a subsequent attempt to run the same Flow with the same
579   * runID will allow the Flow instance to start where it left off.
580   * <p>
581   * Not all planners support this feature.
582   * <p>
583   * A Flow name is required when using a runID.
584   *
585   * @param runID of type String
586   * @return FlowDef
587   */
588  public FlowDef setRunID( String runID )
589    {
590    if( runID != null && runID.isEmpty() )
591      return this;
592
593    this.runID = runID;
594
595    return this;
596    }
597
598  public String getRunID()
599    {
600    return runID;
601    }
602
603  public List<String> getClassPath()
604    {
605    return classPath;
606    }
607
608  /**
609   * Adds each given artifact to the classpath the assembly will execute under allowing
610   * {@link cascading.pipe.Operator}s to dynamically load classes and resources from a {@link ClassLoader}.
611   *
612   * @param artifact a jar or other file String path
613   * @return FlowDef
614   */
615  public FlowDef addToClassPath( String artifact )
616    {
617    if( artifact == null || artifact.isEmpty() )
618      return this;
619
620    classPath.add( artifact );
621
622    return this;
623    }
624  }