    001/*
    002 * Copyright (c) 2016 Chris K Wensel <chris@wensel.net>. All Rights Reserved.
    003 * Copyright (c) 2007-2017 Xplenty, Inc. All Rights Reserved.
    004 *
    005 * Project and contact information: http://www.cascading.org/
    006 *
    007 * This file is part of the Cascading project.
    008 *
    009 * Licensed under the Apache License, Version 2.0 (the "License");
    010 * you may not use this file except in compliance with the License.
    011 * You may obtain a copy of the License at
    012 *
    013 *     http://www.apache.org/licenses/LICENSE-2.0
    014 *
    015 * Unless required by applicable law or agreed to in writing, software
    016 * distributed under the License is distributed on an "AS IS" BASIS,
    017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    018 * See the License for the specific language governing permissions and
    019 * limitations under the License.
    020 */
    021
    022package cascading.flow.stream.element;
    023
    024import java.util.Collection;
    025import java.util.Collections;
    026import java.util.HashSet;
    027import java.util.Iterator;
    028import java.util.Set;
    029
    030import cascading.flow.FlowProcess;
    031import cascading.flow.stream.duct.Duct;
    032import cascading.pipe.Splice;
    033import cascading.tuple.Tuple;
    034import cascading.tuple.TupleEntry;
    035import cascading.tuple.Tuples;
    036
    037/**
    038 *
    039 */
    040public class MemoryCoGroupGate extends MemorySpliceGate
    041  {
    042  public MemoryCoGroupGate( FlowProcess flowProcess, Splice splice )
    043    {
    044    super( flowProcess, splice );
    045    }
    046
    047  @Override
    048  protected boolean isBlockingStreamed()
    049    {
    050    return true;
    051    }
    052
    053  @Override
    054  public void start( Duct previous )
    055    {
    056    }
    057
    058  @Override
    059  public void receive( Duct previous, int ordinal, TupleEntry incomingEntry )
    060    {
    061    Tuple valuesTuple = incomingEntry.getTupleCopy();
    062    Tuple groupTuple = keyBuilder[ ordinal ].makeResult( valuesTuple, null ); // view on valuesTuple
    063
    064    groupTuple = getDelegatedTuple( groupTuple ); // wrap so hasher/comparator is honored
    065
    066    keys.add( groupTuple );
    067    keyValues[ ordinal ].get( groupTuple ).add( valuesTuple );
    068    }
    069
    070  @Override
    071  public void complete( Duct previous )
    072    {
    073    if( count.decrementAndGet() != 0 )
    074      return;
    075
    076    next.start( this );
    077
    078    Collection<Tuple>[] collections = new Collection[ keyValues.length ];
    079    Iterator<Tuple> keyIterator = keys.iterator();
    080
    081    Set<Tuple> seenNulls = new HashSet<Tuple>();
    082
    083    while( keyIterator.hasNext() )
    084      {
    085      Tuple keysTuple = keyIterator.next();
    086
    087      keyIterator.remove();
    088
    089      // provides sql like semantics
    090      if( nullsAreNotEqual && Tuples.frequency( keysTuple, null ) != 0 )
    091        {
    092        if( seenNulls.contains( keysTuple ) )
    093          continue;
    094
    095        seenNulls.add( keysTuple );
    096
    097        for( int i = 0; i < keyValues.length; i++ )
    098          {
    099          Collection<Tuple> values = keyValues[ i ].remove( keysTuple );
    100
    101          if( values == null )
    102            continue;
    103
    104          for( int j = 0; j < keyValues.length; j++ )
    105            collections[ j ] = Collections.emptyList();
    106
    107          collections[ i ] = values;
    108
    109          push( collections, keysTuple );
    110          }
    111        }
    112      else
    113        {
    114        // drain the keys and keyValues collections to preserve memory
    115        for( int i = 0; i < keyValues.length; i++ )
    116          {
    117          collections[ i ] = keyValues[ i ].remove( keysTuple );
    118
    119          if( collections[ i ] == null )
    120            collections[ i ] = Collections.emptyList();
    121          }
    122
    123        push( collections, keysTuple );
    124        }
    125      }
    126
    127    keys = createKeySet();
    128    keyValues = createKeyValuesArray();
    129
    130    count.set( numIncomingEventingPaths );
    131
    132    next.complete( this );
    133    }
    134
    135  private void push( Collection<Tuple>[] collections, Tuple keysTuple )
    136    {
    137    closure.reset( collections );
    138
    139    keyEntry.setTuple( closure.getGroupTuple( keysTuple ) );
    140
    141    // create Closure type here
    142    tupleEntryIterator.reset( splice.getJoiner().getIterator( closure ) );
    143
    144    next.receive( this, 0, grouping );
    145    }
    146  }