001/* 002 * Copyright (c) 2016-2017 Chris K Wensel <chris@wensel.net>. All Rights Reserved. 003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved. 004 * 005 * Project and contact information: http://www.cascading.org/ 006 * 007 * This file is part of the Cascading project. 008 * 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 */ 021 022package cascading.flow; 023 024import java.util.ArrayList; 025import java.util.Arrays; 026import java.util.Collection; 027import java.util.HashMap; 028import java.util.LinkedHashMap; 029import java.util.List; 030import java.util.Map; 031 032import cascading.operation.AssertionLevel; 033import cascading.operation.DebugLevel; 034import cascading.pipe.Checkpoint; 035import cascading.pipe.Pipe; 036import cascading.property.UnitOfWorkDef; 037import cascading.tap.Tap; 038import cascading.util.Util; 039 040/** 041 * Class FlowDef is a fluent interface for defining a {@link Flow}. 042 * <p> 043 * This allows for ad-hoc building of Flow data and meta-data, like tags. 044 * <p> 045 * Instead of calling one of the {@link FlowConnector} connect methods, {@link FlowConnector#connect(FlowDef)} 046 * can be called. 047 */ 048public class FlowDef extends UnitOfWorkDef<FlowDef> 049 { 050 protected Map<String, Tap> sources = new HashMap<String, Tap>(); 051 protected Map<String, Tap> sinks = new HashMap<String, Tap>(); 052 protected Map<String, Tap> traps = new HashMap<String, Tap>(); 053 protected Map<String, Tap> checkpoints = new HashMap<String, Tap>(); 054 055 protected List<String> classPath = new ArrayList<String>(); 056 protected List<Pipe> tails = new ArrayList<Pipe>(); 057 protected List<AssemblyPlanner> assemblyPlanners = new ArrayList<AssemblyPlanner>(); 058 059 protected HashMap<String, String> flowDescriptor = new LinkedHashMap<String, String>(); 060 061 protected AssertionLevel assertionLevel; 062 protected DebugLevel debugLevel; 063 064 protected String runID; 065 066 /** 067 * Creates a new instance of a FlowDef. 068 * 069 * @return a FlowDef 070 */ 071 public static FlowDef flowDef() 072 { 073 return new FlowDef(); 074 } 075 076 /** Constructor FlowDef creates a new FlowDef instance. */ 077 public FlowDef() 078 { 079 } 080 081 protected FlowDef( FlowDef flowDef, Map<String, Tap> sources, Map<String, Tap> sinks, Map<String, Tap> traps, Map<String, Tap> checkpoints ) 082 { 083 super( flowDef ); 084 085 this.sources = sources; 086 this.sinks = sinks; 087 this.traps = traps; 088 this.checkpoints = checkpoints; 089 090 this.classPath = flowDef.classPath; 091 this.tails = flowDef.tails; 092 this.assemblyPlanners = flowDef.assemblyPlanners; 093 this.flowDescriptor = flowDef.flowDescriptor; 094 this.assertionLevel = flowDef.assertionLevel; 095 this.debugLevel = flowDef.debugLevel; 096 this.runID = flowDef.runID; 097 } 098 099 /** 100 * Method getAssemblyPlanners returns the current registered AssemblyPlanners. 101 * 102 * @return a List of AssemblyPlanner instances 103 */ 104 public List<AssemblyPlanner> getAssemblyPlanners() 105 { 106 return assemblyPlanners; 107 } 108 109 /** 110 * Method addAssemblyPlanner adds new AssemblyPlanner instances to be evaluated. 111 * 112 * @param assemblyPlanner of type AssemblyPlanner 113 * @return a FlowDef 114 */ 115 public FlowDef addAssemblyPlanner( AssemblyPlanner assemblyPlanner ) 116 { 117 assemblyPlanners.add( assemblyPlanner ); 118 addDescriptions( assemblyPlanner.getFlowDescriptor() ); 119 120 return this; 121 } 122 123 /** 124 * Method getSources returns the sources of this FlowDef object. 125 * 126 * @return the sources (type Map) of this FlowDef object. 127 */ 128 public Map<String, Tap> getSources() 129 { 130 return sources; 131 } 132 133 /** 134 * Method getSourcesCopy returns a copy of the sources Map. 135 * 136 * @return the sourcesCopy (type Map) of this FlowDef object. 137 */ 138 public Map<String, Tap> getSourcesCopy() 139 { 140 return new HashMap<String, Tap>( sources ); 141 } 142 143 /** 144 * Method getFlowDescriptor returns the flowDescriptor of this FlowDef. 145 * 146 * @return the flowDescriptor of this FlowDef object. 147 */ 148 public HashMap<String, String> getFlowDescriptor() 149 { 150 return flowDescriptor; 151 } 152 153 /** 154 * Method addSource adds a new named source {@link Tap} for use in the resulting {@link Flow}. 155 * 156 * @param name of String 157 * @param source of Tap 158 * @return FlowDef 159 */ 160 public FlowDef addSource( String name, Tap source ) 161 { 162 if( sources.containsKey( name ) ) 163 throw new IllegalArgumentException( "cannot add duplicate source: " + name ); 164 165 sources.put( name, source ); 166 return this; 167 } 168 169 /** 170 * Method addSource adds a new source {@link Tap} named after the given {@link Pipe} for use in the resulting {@link Flow}. 171 * <p> 172 * If the given pipe is not a head pipe, it will be resolved. If more than one is found, an 173 * {@link IllegalArgumentException} will be thrown. 174 * 175 * @param pipe of Pipe 176 * @param source of Tap 177 * @return FlowDef 178 */ 179 public FlowDef addSource( Pipe pipe, Tap source ) 180 { 181 if( pipe == null ) 182 throw new IllegalArgumentException( "pipe may not be null" ); 183 184 Pipe[] heads = pipe.getHeads(); 185 186 if( heads.length != 1 ) 187 throw new IllegalArgumentException( "pipe has too many heads, found: " + Arrays.toString( Pipe.names( heads ) ) ); 188 189 addSource( heads[ 0 ].getName(), source ); 190 return this; 191 } 192 193 /** 194 * Method addSources adds a map of name and {@link Tap} pairs. 195 * 196 * @param sources of Map 197 * @return FlowDef 198 */ 199 public FlowDef addSources( Map<String, Tap> sources ) 200 { 201 if( sources != null ) 202 { 203 for( Map.Entry<String, Tap> entry : sources.entrySet() ) 204 addSource( entry.getKey(), entry.getValue() ); 205 } 206 207 return this; 208 } 209 210 /** 211 * Method addDescription adds a user readable description to the flowDescriptor. 212 * <p> 213 * This uses the {@link FlowDescriptors#DESCRIPTION} key. 214 */ 215 public FlowDef addDescription( String description ) 216 { 217 addDescription( FlowDescriptors.DESCRIPTION, description ); 218 219 return this; 220 } 221 222 /** 223 * Method addDescription adds a description to the flowDescriptor. 224 * <p> 225 * Flow descriptions provide meta-data to monitoring systems describing the workload a given Flow represents. 226 * For known description types, see {@link FlowDescriptors}. 227 * <p> 228 * If an existing key exists, it will be appended to the original value using 229 * {@link FlowDescriptors#VALUE_SEPARATOR}. 230 * 231 * @param key The key as a String. 232 * @param value The value as a String. 233 * @return FlowDef 234 */ 235 public FlowDef addDescription( String key, String value ) 236 { 237 if( Util.isEmpty( value ) ) // do nothing 238 return this; 239 240 if( flowDescriptor.containsKey( key ) ) 241 { 242 String original = flowDescriptor.get( key ); 243 244 if( !Util.isEmpty( original ) ) 245 value = original + FlowDescriptors.VALUE_SEPARATOR + value; 246 } 247 248 flowDescriptor.put( key, value ); 249 250 return this; 251 } 252 253 /** 254 * Method addProperties adds all properties in the given map in order to the flowDescriptor. If the given Map has 255 * an explicit order, it will be preserved. 256 * <p> 257 * Flow descriptions provide meta-data to monitoring systems describing the workload a given Flow represents. 258 * For known description types, see {@link FlowDescriptors}. 259 * 260 * @param descriptions The properties to be added to the map. 261 * @return FlowDef 262 */ 263 public FlowDef addDescriptions( Map<String, String> descriptions ) 264 { 265 for( Map.Entry<String, String> entry : descriptions.entrySet() ) 266 addDescription( entry.getKey(), entry.getValue() ); 267 268 return this; 269 } 270 271 /** 272 * Method getSinks returns the sinks of this FlowDef object. 273 * 274 * @return the sinks (type Map) of this FlowDef object. 275 */ 276 public Map<String, Tap> getSinks() 277 { 278 return sinks; 279 } 280 281 /** 282 * Method getSinksCopy returns a copy of the sink Map. 283 * 284 * @return the sinksCopy (type Map) of this FlowDef object. 285 */ 286 public Map<String, Tap> getSinksCopy() 287 { 288 return new HashMap<String, Tap>( sinks ); 289 } 290 291 /** 292 * Method addSink adds a new named sink {@link Tap} for use in the resulting {@link Flow}. 293 * 294 * @param name of String 295 * @param sink of Tap 296 * @return FlowDef 297 */ 298 public FlowDef addSink( String name, Tap sink ) 299 { 300 if( sinks.containsKey( name ) ) 301 throw new IllegalArgumentException( "cannot add duplicate sink: " + name ); 302 303 sinks.put( name, sink ); 304 return this; 305 } 306 307 /** 308 * Method addSink adds a new sink {@link Tap} named after the given {@link Pipe} for use in the resulting {@link Flow}. 309 * 310 * @param tail of Pipe 311 * @param sink of Tap 312 * @return FlowDef 313 */ 314 public FlowDef addSink( Pipe tail, Tap sink ) 315 { 316 addSink( tail.getName(), sink ); 317 return this; 318 } 319 320 /** 321 * Method addTailSink adds the tail {@link Pipe} and sink {@link Tap} to this FlowDef. 322 * <p> 323 * This is a convenience method for adding both a tail and sink simultaneously. There isn't a similar method 324 * for heads and sources as the head Pipe can always be derived. 325 * 326 * @param tail of Pipe 327 * @param sink of Tap 328 * @return FlowDef 329 */ 330 public FlowDef addTailSink( Pipe tail, Tap sink ) 331 { 332 addSink( tail.getName(), sink ); 333 addTail( tail ); 334 return this; 335 } 336 337 /** 338 * Method addSinks adds a Map of the named and {@link Tap} pairs. 339 * 340 * @param sinks of Map 341 * @return FlowDef 342 */ 343 public FlowDef addSinks( Map<String, Tap> sinks ) 344 { 345 if( sinks != null ) 346 { 347 for( Map.Entry<String, Tap> entry : sinks.entrySet() ) 348 addSink( entry.getKey(), entry.getValue() ); 349 } 350 351 return this; 352 } 353 354 /** 355 * Method getTraps returns the traps of this FlowDef object. 356 * 357 * @return the traps (type Map) of this FlowDef object. 358 */ 359 public Map<String, Tap> getTraps() 360 { 361 return traps; 362 } 363 364 /** 365 * Method getTrapsCopy returns a copy of the trap Map. 366 * 367 * @return the trapsCopy (type Map) of this FlowDef object. 368 */ 369 public Map<String, Tap> getTrapsCopy() 370 { 371 return new HashMap<String, Tap>( traps ); 372 } 373 374 /** 375 * Method addTrap adds a new named trap {@link Tap} for use in the resulting {@link Flow}. 376 * 377 * @param name of String 378 * @param trap of Tap 379 * @return FlowDef 380 */ 381 public FlowDef addTrap( String name, Tap trap ) 382 { 383 if( traps.containsKey( name ) ) 384 throw new IllegalArgumentException( "cannot add duplicate trap: " + name ); 385 386 traps.put( name, trap ); 387 return this; 388 } 389 390 /** 391 * Method addTrap adds a new trap {@link Tap} named after the given {@link Pipe} for use in the resulting {@link Flow}. 392 * 393 * @param pipe of Pipe 394 * @param trap of Tap 395 * @return FlowDef 396 */ 397 public FlowDef addTrap( Pipe pipe, Tap trap ) 398 { 399 addTrap( pipe.getName(), trap ); 400 return this; 401 } 402 403 /** 404 * Method addTraps adds a Map of the names and {@link Tap} pairs. 405 * 406 * @param traps of Map 407 * @return FlowDef 408 */ 409 public FlowDef addTraps( Map<String, Tap> traps ) 410 { 411 if( traps != null ) 412 { 413 for( Map.Entry<String, Tap> entry : traps.entrySet() ) 414 addTrap( entry.getKey(), entry.getValue() ); 415 } 416 417 return this; 418 } 419 420 /** 421 * Method getCheckpoints returns the checkpoint taps of this FlowDef object. 422 * 423 * @return the checkpoints (type Map) of this FlowDef object. 424 */ 425 public Map<String, Tap> getCheckpoints() 426 { 427 return checkpoints; 428 } 429 430 /** 431 * Method getCheckpointsCopy returns a copy of the checkpoint tap Map. 432 * 433 * @return the checkpointsCopy (type Map) of this FlowDef object. 434 */ 435 public Map<String, Tap> getCheckpointsCopy() 436 { 437 return new HashMap<String, Tap>( checkpoints ); 438 } 439 440 /** 441 * Method addCheckpoint adds a new named checkpoint {@link Tap} for use in the resulting {@link Flow}. 442 * 443 * @param name of String 444 * @param checkpoint of Tap 445 * @return FlowDef 446 */ 447 public FlowDef addCheckpoint( String name, Tap checkpoint ) 448 { 449 if( checkpoints.containsKey( name ) ) 450 throw new IllegalArgumentException( "cannot add duplicate checkpoint: " + name ); 451 452 checkpoints.put( name, checkpoint ); 453 return this; 454 } 455 456 /** 457 * Method addCheckpoint adds a new checkpoint {@link Tap} named after the given {@link Checkpoint} for use in the resulting {@link Flow}. 458 * 459 * @param pipe of Pipe 460 * @param checkpoint of Tap 461 * @return FlowDef 462 */ 463 public FlowDef addCheckpoint( Checkpoint pipe, Tap checkpoint ) 464 { 465 addCheckpoint( pipe.getName(), checkpoint ); 466 return this; 467 } 468 469 /** 470 * Method addCheckpoints adds a Map of the names and {@link Tap} pairs. 471 * 472 * @param checkpoints of Map 473 * @return FlowDef 474 */ 475 public FlowDef addCheckpoints( Map<String, Tap> checkpoints ) 476 { 477 if( checkpoints != null ) 478 { 479 for( Map.Entry<String, Tap> entry : checkpoints.entrySet() ) 480 addCheckpoint( entry.getKey(), entry.getValue() ); 481 } 482 483 return this; 484 } 485 486 /** 487 * Method getTails returns all the current pipe assembly tails the FlowDef holds. 488 * 489 * @return the tails (type List) of this FlowDef object. 490 */ 491 public List<Pipe> getTails() 492 { 493 return tails; 494 } 495 496 /** 497 * Method getTailsArray returns all the current pipe assembly tails the FlowDef holds. 498 * 499 * @return the tailsArray (type Pipe[]) of this FlowDef object. 500 */ 501 public Pipe[] getTailsArray() 502 { 503 return tails.toArray( new Pipe[ tails.size() ] ); 504 } 505 506 /** 507 * Method addTail adds a new {@link Pipe} to this FlowDef that represents a tail in a pipe assembly. 508 * <p> 509 * Be sure to add a sink tap that has the same name as this tail. 510 * 511 * @param tail of Pipe 512 * @return FlowDef 513 */ 514 public FlowDef addTail( Pipe tail ) 515 { 516 if( tail != null ) 517 this.tails.add( tail ); 518 519 return this; 520 } 521 522 /** 523 * Method addTails adds a Collection of tails. 524 * 525 * @param tails of Collection 526 * @return FlowDef 527 */ 528 public FlowDef addTails( Collection<Pipe> tails ) 529 { 530 for( Pipe tail : tails ) 531 addTail( tail ); 532 533 return this; 534 } 535 536 /** 537 * Method addTails adds an array of tails. 538 * 539 * @param tails of Pipe... 540 * @return FlowDef 541 */ 542 public FlowDef addTails( Pipe... tails ) 543 { 544 for( Pipe tail : tails ) 545 addTail( tail ); 546 547 return this; 548 } 549 550 public FlowDef setAssertionLevel( AssertionLevel assertionLevel ) 551 { 552 this.assertionLevel = assertionLevel; 553 554 return this; 555 } 556 557 public AssertionLevel getAssertionLevel() 558 { 559 return assertionLevel; 560 } 561 562 public FlowDef setDebugLevel( DebugLevel debugLevel ) 563 { 564 this.debugLevel = debugLevel; 565 566 return this; 567 } 568 569 public DebugLevel getDebugLevel() 570 { 571 return debugLevel; 572 } 573 574 /** 575 * Method setRunID sets the checkpoint run or execution ID to be used to find prior failed runs against 576 * this runID. 577 * <p> 578 * When given, and a {@link Flow} fails to execute, a subsequent attempt to run the same Flow with the same 579 * runID will allow the Flow instance to start where it left off. 580 * <p> 581 * Not all planners support this feature. 582 * <p> 583 * A Flow name is required when using a runID. 584 * 585 * @param runID of type String 586 * @return FlowDef 587 */ 588 public FlowDef setRunID( String runID ) 589 { 590 if( runID != null && runID.isEmpty() ) 591 return this; 592 593 this.runID = runID; 594 595 return this; 596 } 597 598 public String getRunID() 599 { 600 return runID; 601 } 602 603 public List<String> getClassPath() 604 { 605 return classPath; 606 } 607 608 /** 609 * Adds each given artifact to the classpath the assembly will execute under allowing 610 * {@link cascading.pipe.Operator}s to dynamically load classes and resources from a {@link ClassLoader}. 611 * 612 * @param artifact a jar or other file String path 613 * @return FlowDef 614 */ 615 public FlowDef addToClassPath( String artifact ) 616 { 617 if( artifact == null || artifact.isEmpty() ) 618 return this; 619 620 classPath.add( artifact ); 621 622 return this; 623 } 624 }