TPIE

2362a60
tpie::pipelining Namespace Reference

TPIE pipelining framework. More...

Classes

class  any_noncopyable
 
class  Args
 
class  bad_any_noncopyable_cast
 
class  bad_any_noncopyable_copy
 
class  call_order_exception
 
class  container
 
struct  container< T, TT...>
 
struct  destination_kind
 
class  empty_pipe_middle
 A empty_pipe_middle class has no effect. More...
 
class  factory
 Node factory for variadic argument generators. More...
 
class  factory_base
 Base class of all pipelining factories. More...
 
class  factory_init_hook
 
class  internal_passive_buffer
 Internal fifo buffer. More...
 
class  internal_passive_reverser
 A passive reverser stored in internal memory. More...
 
class  join
 Joins multiple push streams into one. More...
 
class  no_initiator_node
 
class  node
 Base class of all nodes. More...
 
struct  node_parameters
 
struct  node_resource_parameters
 
class  node_token
 
class  non_authoritative_node_map
 
class  not_initiator_node
 
class  passive_buffer
 Plain old file_stream buffer. More...
 
class  passive_reverser
 A passive reverser stored in external memory. More...
 
class  passive_serialization_buffer
 Serialization stream buffer. More...
 
class  passive_serialization_reverser
 A passive serialization reverser stored in external memory. More...
 
class  passive_sorter
 Pipelined sorter with push input and pull output. More...
 
class  pipe_begin
 
class  pipe_end
 
class  pipe_middle
 A pipe_middle class pushes input down the pipeline. More...
 
class  pipeline
 This class is used to avoid writing the template argument in the pipeline_impl type. More...
 
struct  pull_type
 
class  pullpipe_begin
 
class  pullpipe_end
 
class  pullpipe_middle
 
struct  push_type
 Class to deduce the item_type of a node of type T. More...
 
class  serialization_passive_sorter
 Pipelined sorter with push input and pull output. More...
 
class  split
 Split one push streams into multiple. More...
 
class  split_factory
 Node factory for split nodes, typically used for phase boundary nodes. More...
 
struct  subpipeline
 
class  tempfactory
 Node factory for variadic argument templated generators. More...
 
class  termfactory
 Node factory for variadic argument terminators. More...
 
class  tfactory
 Node factory for variadic argument terminators. More...
 
class  tfactory< R, Args< TT...>, T...>
 
class  virtual_chunk
 Virtual chunk that has input and output. More...
 
class  virtual_chunk_begin
 Virtual chunk that has no input (that is, virtual producer). More...
 
class  virtual_chunk_end
 Virtual chunk that has no output (that is, virtual consumer). More...
 
class  virtual_chunk_missing_begin
 
class  virtual_chunk_missing_end
 
class  virtual_chunk_missing_middle
 
class  virtual_chunk_not_ready
 
class  virtual_container
 Virtual base class for extra data to go with virtual chunks. More...
 

Typedefs

typedef pipe_middle
< split_factory
< bits::buffer_input_t, node,
bits::buffer_output_t > > 
buffer
 The buffer node inserts a phase boundary into the pipeline by writing items to disk. More...
 
typedef pipe_middle< factory
< bits::chunker_t, size_t > > 
chunker
 A pipelining node that gathers elements into a vector of some size. More...
 
typedef tpie::flags
< stream_option > 
stream_options
 
typedef pipe_begin< factory
< bits::named_input_t,
std::string > > 
named_input
 Pipelining nodes that pushes the contents of the named file stream to the next node in the pipeline. More...
 
typedef pullpipe_middle
< factory< bits::pull_peek_t > > 
pull_peek
 A node that allows peeking at the next item in the pipeline. More...
 
typedef boost::intrusive_ptr
< bits::node_set_content
node_set
 
typedef int priority_type
 
typedef pipe_middle
< split_factory
< bits::reverser_input_t, node,
bits::reverser_output_t > > 
reverser
 Constructs a reverser node stored in external memory. More...
 
typedef pipe_middle
< split_factory
< bits::internal_reverser_input_t,
node,
bits::internal_reverser_output_t > > 
internal_reverser
 Constructs a reverser node stored in internal memory. More...
 
typedef pipe_middle
< split_factory
< serialization_bits::reverser_input_t,
node,
serialization_bits::reverser_output_t > > 
serialization_reverser
 A pipelining node that reverses serializable items and creates a phase boundary. More...
 
typedef pipe_middle
< split_factory
< serialization_bits::buffer_input_t,
node,
serialization_bits::buffer_output_t > > 
serialization_buffer
 A pipelining node that acts as a buffer for serializable items and creates a phase boundary. More...
 
typedef pipe_begin< factory
< bits::scanf_ints_t > > 
scanf_ints
 A pipelining node that pushes the integers it reads using scanf. More...
 
typedef pipe_end< termfactory
< bits::printf_ints_t > > 
printf_ints
 A pipelining node that prints the items that are pushed to it. More...
 

Enumerations

enum  stream_option { STREAM_RESET =1, STREAM_CLOSE =2 }
 
enum  maintain_order_type { arbitrary_order = false, maintain_order = true }
 Type describing whether to maintain the order of items in parallel. More...
 

Functions

template<typename T >
pipe_begin< factory
< bits::ami_input_t,
tpie::ami::stream< T > & > > 
ami_input (tpie::ami::stream< T > &input)
 Pipelining nodes that pushes the contents of the given ami::stream to the next node in the pipeline. More...
 
template<typename T >
pipe_begin< factory
< bits::ami_input_stack_t,
tpie::ami::stack< T > & > > 
ami_input_stack (tpie::ami::stack< T > &input)
 Pipelining nodes that pushes the contents of the given ami::stack to the next node in the pipeline. More...
 
template<typename T >
pullpipe_begin< termfactory
< bits::pull_input_t< T >
, tpie::ami::stack< T > & > > 
ami_pull_input_stack (tpie::ami::stack< T > &fs)
 A pipelining pull-node that reads items from the given ami::stack. More...
 
template<typename T >
pipe_end< termfactory
< bits::ami_output_t< T >
, tpie::ami::stream< T > & > > 
ami_output (tpie::ami::stream< T > &fs)
 A pipelining node that writes the pushed items to an ami stream. More...
 
template<typename T >
T & move_if_movable (typename std::remove_reference< T >::type &t, typename std::enable_if<!std::is_move_constructible< T >::value >::type *=0)
 
template<typename T >
T && move_if_movable (typename std::remove_reference< T >::type &t, typename std::enable_if< std::is_move_constructible< T >::value >::type *=0)
 
template<typename T >
T & move_if_movable_rvalue (typename std::remove_reference< T >::type &t, typename std::enable_if< !std::is_move_constructible< T >::value||!std::is_rvalue_reference< T >::value >::type *=0)
 
template<typename T >
T && move_if_movable_rvalue (typename std::remove_reference< T >::type &t, typename std::enable_if< std::is_move_constructible< T >::value &&std::is_rvalue_reference< T >::value >::type *=0)
 
template<int i, typename... T>
bits::get_impl< i, T...>::type & get (container< T...> &c)
 
template<int i, typename... T>
const bits::get_impl< i, T...>
::type & 
get (const container< T...> &c)
 
template<typename F , typename... T1, typename... T2>
container_construct (container< T1...> &cont, T2 &&...a)
 
template<typename F , typename... T1, typename... T2>
container_construct_copy (container< T1...> &cont, T2 &&...a)
 
template<typename T >
const T & any_cast (const any_noncopyable &a)
 
template<typename T >
T & any_cast (any_noncopyable &a)
 
void swap (any_noncopyable &l, any_noncopyable &r)
 
template<typename T >
pipe_begin< factory
< bits::input_t, file_stream
< T > &, stream_options > > 
input (file_stream< T > &fs, stream_options options=stream_options())
 Pipelining nodes that pushes the contents of the given file stream to the next node in the pipeline. More...
 
template<typename T >
pullpipe_begin< termfactory
< bits::pull_input_t< T >
, file_stream< T >
&, stream_options > > 
pull_input (file_stream< T > &fs, stream_options options=stream_options())
 A pipelining pull-node that reads items from the given file_stream. More...
 
template<typename T >
pullpipe_begin< termfactory
< bits::pull_reverse_input_t
< T >, file_stream< T >
&, stream_options > > 
pull_reverse_input (file_stream< T > &fs, stream_options options=stream_options())
 A pipelining pull-node that reads items in reverse order from the given file_stream. More...
 
template<typename T >
pullpipe_begin< termfactory
< bits::named_pull_input_t< T >
, std::string > > 
named_pull_input (std::string name)
 A pipelining pull-node that reads items from the given file_stream. More...
 
template<typename T >
pipe_end< termfactory
< bits::output_t< T >
, file_stream< T > & > > 
output (file_stream< T > &fs)
 A pipelining node that writes the pushed items to a file stream. More...
 
template<typename T >
pipe_end< termfactory
< bits::named_output_t< T >
, std::string > > 
named_output (std::string path)
 A pipelining node that writes the pushed items to a named file stream. More...
 
template<typename T >
pullpipe_end< factory
< bits::pull_output_t,
file_stream< T > & > > 
pull_output (file_stream< T > &fs)
 A pull-pipe node that writes the pulled items to a file stream. More...
 
template<typename T >
pipe_middle< tfactory
< bits::tee_t, Args< typename
T::item_type >, T & > > 
tee (T &fs)
 A pipelining node that writes the pushed to a file stream and then pushes the items to the next node. More...
 
template<typename T >
pullpipe_middle< tfactory
< bits::pull_tee_t, Args
< typename T::item_type >, T & > > 
pull_tee (T &fs)
 A pull-pipe node that when pulled from will pull from its source, write its item to disk and then return the item. More...
 
template<typename F >
pipe_middle< tempfactory
< bits::filter_t< F >, F > > 
filter (const F &functor)
 A pipelining node that keeps only elements where functor evaluates to true. More...
 
pipe_middle< factory
< bits::Forwarder, std::vector
< std::pair< std::string,
any_noncopyable > > > > 
forwarder (std::vector< std::pair< std::string, any_noncopyable > > items)
 A pipelining node that will forward values on prepare, and. More...
 
template<typename VT >
pipe_middle< factory
< bits::Forwarder, std::vector
< std::pair< std::string,
any_noncopyable > > > > 
forwarder (std::string name, VT value)
 A pipelining node that will forward value on prepare, and. More...
 
pipe_middle< factory
< bits::ostream_logger_t,
std::ostream & > > 
cout_logger ()
 A pipelining node that writes items to standard out and then pushes them to the next node. More...
 
template<typename fact_t >
pipe_middle< tempfactory
< bits::fork_t< fact_t >
, fact_t > > 
fork (pipe_end< fact_t > to)
 Create a fork pipe node. More...
 
template<typename dest_fact_t >
pullpipe_middle< tfactory
< bits::pull_fork_t, Args
< dest_fact_t >, dest_fact_t > > 
pull_fork (dest_fact_t dest_fact)
 Create a pulling fork pipe node. More...
 
template<typename fact_t >
pipe_middle< tempfactory
< bits::unzip_t< fact_t >
, fact_t > > 
unzip (pipe_end< fact_t > to)
 Create unzip pipe node. More...
 
template<typename fact_t >
pipe_middle< tempfactory
< bits::zip_t< fact_t >
, fact_t > > 
zip (pullpipe_begin< fact_t > from)
 Create a zip pipe node. More...
 
template<typename T >
pipe_end< termfactory
< bits::null_sink_t< T > > > 
null_sink ()
 Create a dummy end pipe node. More...
 
template<typename T >
pullpipe_begin< termfactory
< bits::zero_source_t< T > > > 
zero_source ()
 Create a dummy pull begin pipe node. More...
 
template<template< typename dest_t > class Fact, typename... T>
pipe_begin< factory< Fact, T...> > make_pipe_begin (T...t)
 
template<template< typename dest_t > class Fact, typename... T>
pipe_middle< factory< Fact, T...> > make_pipe_middle (T...t)
 
template<typename Fact , typename... T>
pipe_end< termfactory< Fact,
T...> > 
make_pipe_end (T...t)
 
template<typename IT >
pullpipe_begin< termfactory
< bits::pull_input_iterator_t
< IT >, IT, IT > > 
pull_input_iterator (IT begin, IT end)
 A pull-pipe that returns items in the range given by two iterators. More...
 
template<typename IT >
pipe_begin< tempfactory
< bits::push_input_iterator_t
< IT >, IT, IT > > 
push_input_iterator (IT begin, IT end)
 A pipelining node that pushes the items in the range given by two iterators. More...
 
template<typename IT >
pipe_end< termfactory
< bits::push_output_iterator_t
< IT >, IT > > 
push_output_iterator (IT to)
 A node that writes its given items to the destination pointed to by the given iterator. More...
 
template<typename Item , typename IT >
pipe_end< termfactory
< bits::push_output_iterator_t
< IT, Item >, IT > > 
typed_push_output_iterator (IT to)
 A node that writes its given items to the destination pointed to by the given iterator. More...
 
template<typename IT >
pullpipe_end< tempfactory
< bits::pull_output_iterator_t
< IT >, IT > > 
pull_output_iterator (IT to)
 A pull-pipe node that writes its given items to the destination pointed to by the given iterator. More...
 
template<typename F >
pipe_middle< tempfactory
< bits::preparer_t< F >, F > > 
preparer (const F &functor)
 Create preparer callback identity pipe node. More...
 
template<typename F >
pipe_middle< tempfactory
< bits::propagater_t< F >, F > > 
propagater (const F &functor)
 Create propagate callback identity pipe node. More...
 
template<typename T >
pipe_middle< tempfactory
< bits::item_type_t< T > > > 
item_type ()
 Create item type defining identity pipe node. More...
 
template<typename fact_t >
pipe_begin< tempfactory
< bits::pull_source_t< fact_t >
, fact_t > > 
pull_source (pullpipe_begin< fact_t > from)
 A node that pulls items from source and push them into dest. More...
 
template<typename equal_t >
pipe_middle< tfactory
< bits::unique_t, Args
< equal_t >, equal_t > > 
unique (equal_t equal)
 Filter consecutive duplicates out. More...
 
template<typename F , typename = typename std::enable_if<bits::has_argument_type<F>::value>::type>
pipe_middle< tempfactory
< bits::map_t< F >, F > > 
map (const F &functor)
 Pipelining nodes that applies to given functor to items in the stream. More...
 
template<typename F , typename = typename std::enable_if<!bits::has_argument_type<F>::value>::type>
pipe_middle< tempfactory
< bits::map_temp_t< F >, F > > 
map (const F &functor)
 
template<typename F >
pipe_end< termfactory
< bits::map_sink_t< F >, F > > 
map_sink (const F &functor)
 
template<typename pull_t >
pipe_middle< factory
< bits::merge_t< pull_t >
::template type, pull_t > > 
merge (pullpipe_begin< pull_t > with)
 A node that merges a pull pipeline into a push pipeline. More...
 
 TPIE_DECLARE_OPERATORS_FOR_FLAGS (node::PLOT)
 
node_set make_node_set ()
 
template<typename T >
pipe_middle< factory
< bits::linear_t, T, T > > 
linear (T factor, T term)
 A pipelining node that transforms the items by applying a linear function to them. More...
 
template<typename T >
pipe_begin< factory
< bits::range_t, T, T, T > > 
range (T from, T to, T increment=1)
 
template<typename fact_t >
pipe_middle
< parallel_bits::factory
< fact_t > > 
parallel (pipe_middle< fact_t > &&fact, maintain_order_type maintainOrder, size_t numJobs, size_t bufSize=2048)
 Runs a pipeline in multiple threads. More...
 
template<typename fact_t >
pipe_middle
< parallel_bits::factory
< fact_t > > 
parallel (pipe_middle< fact_t > &&fact, maintain_order_type maintainOrder=arbitrary_order)
 Runs a pipeline in multiple threads, using the number of threads reported by tpie::default_worker_count. More...
 
template<typename fact_t >
pipe_middle
< parallel_bits::factory
< fact_t > > 
parallel (pipe_middle< fact_t > &&fact, bool maintainOrder, size_t numJobs, size_t bufSize=2048)
 
template<typename fact_t >
pipe_middle
< parallel_bits::factory
< fact_t > > 
parallel (pipe_middle< fact_t > &&fact, bool maintainOrder)
 
pipe_begin
< serialization_bits::input_factory
serialization_input (serialization_reader &rd)
 A pipelining node that reads items from a serialization_reader. More...
 
template<typename T >
pipe_end< typename
serialization_bits::output_factory
< T >::type > 
serialization_output (serialization_writer &wr)
 A pipelining node that writes item to a serialization_writer. More...
 
pipe_middle
< serialization_bits::default_pred_sort_factory
serialization_sort ()
 Pipelining sorter using std::less. More...
 
template<typename pred_t >
pipe_middle
< serialization_bits::sort_factory
< pred_t > > 
serialization_sort (const pred_t &p)
 Pipelining sorter using the given predicate. More...
 
pipe_middle
< bits::default_pred_sort_factory
< default_store > > 
sort ()
 Pipelining sorter using std::less. More...
 
template<typename store_t >
pipe_middle
< bits::default_pred_sort_factory
< store_t > > 
store_sort (store_t store=store_t())
 A pipelining node that sorts large elements indirectly by using a store and std::less. More...
 
template<typename pred_t >
pipe_middle
< bits::sort_factory< pred_t,
default_store > > 
sort (const pred_t &p)
 Pipelining sorter using the given predicate. More...
 
template<typename pred_t , typename store_t >
pipe_middle
< bits::sort_factory< pred_t,
store_t > > 
sort (const pred_t &p, store_t store)
 A pipelining node that sorts large elements indirectly by using a storeand a given predicate. More...
 
template<typename T , typename A >
pipe_begin< tfactory
< bits::input_vector_t, Args
< T, A >, const std::vector< T,
A > & > > 
input_vector (const std::vector< T, A > &input)
 Pipelining nodes that pushes the contents of the given vector to the next node in the pipeline. More...
 
template<typename T , typename A >
pipe_begin< tfactory
< bits::input_vector_t, Args
< T, A >, const std::vector< T,
A > & > > 
input_vector (std::vector< T, A > &&input)=delete
 
template<typename T , typename A >
pullpipe_begin< termfactory
< bits::pull_input_vector_t< T,
A >, const std::vector< T, A > & > > 
pull_input_vector (const std::vector< T, A > &input)
 Pipelining nodes that pushes the contents of the given vector to the next node in the pipeline. More...
 
template<typename T , typename A >
pullpipe_begin< termfactory
< bits::pull_input_vector_t< T,
A >, const std::vector< T, A > & > > 
pull_input_vector (std::vector< T, A > &&input)=delete
 
template<typename T , typename A >
pipe_end< termfactory
< bits::output_vector_t< T, A >
, std::vector< T, A > & > > 
output_vector (std::vector< T, A > &output)
 Pipelining node that pushes items to the given vector. More...
 
template<typename T , typename A >
pipe_end< termfactory
< bits::output_vector_t< T, A >
, std::vector< T, A > & > > 
output_vector (std::vector< T, A > &&output)=delete
 
template<typename F >
pipe_middle< tempfactory
< bits::lambda_t< F >, F > > 
lambda (const F &f)
 Pipelining nodes that applies to given functor to items in the stream. More...
 
template<typename F >
pipe_middle< tempfactory
< bits::exclude_lambda_t< F >
, F > > 
exclude_lambda (const F &f)
 Pipelining nodes that applies to given functor to items in the stream. More...
 
pipe_middle
< bits::pair_factory< factory
< bits::count_consecutive_t >
, factory
< bits::extract_first_t > > > 
pipeuniq ()
 A pipelining node that removes duplicate items and create a phase boundary. More...
 
template<typename T >
pipe_middle< tempfactory
< bits::vfork_node< T >
, virtual_chunk_end< T > > > 
fork_to_virtual (const virtual_chunk_end< T > &out)
 
template<typename T >
pipe_end< termfactory
< bits::vpush_node< T >
, virtual_chunk_end< T > > > 
push_to_virtual (const virtual_chunk_end< T > &out)
 
template<typename T >
virtual_chunk< T > vfork (const virtual_chunk_end< T > &out)
 
template<typename T >
virtual_chunk< T > chunk_if (bool b, virtual_chunk< T > t)
 
template<typename T >
virtual_chunk_end< T > chunk_end_if (bool b, virtual_chunk_end< T > t)
 
template<typename F >
pipe_middle< tempfactory
< bits::visit_t< F >, F > > 
visit (const F &functor)
 A pipelining node that applies a functor to elements pushed to it. More...
 

Variables

std::unordered_set
< bits::pipeline_base_base * > 
current_pipelines
 
std::mutex current_pipelines_mutex
 
const priority_type PRIORITY_NO_NAME = 0
 
const priority_type PRIORITY_INSIGNIFICANT = 5
 
const priority_type PRIORITY_SIGNIFICANT = 10
 
const priority_type PRIORITY_USER = 20
 

Detailed Description

TPIE pipelining framework.

Author
Mathias Rav

Typedef Documentation

The buffer node inserts a phase boundary into the pipeline by writing items to disk.

It does not change the contents of the stream.

Definition at line 207 of file buffer.h.

A pipelining node that gathers elements into a vector of some size.

Parameters
maxSizethe maximum size of the vector

Definition at line 76 of file chunker.h.

Constructs a reverser node stored in internal memory.

Reverses the stream and creates a phase boundary

Definition at line 376 of file reverse.h.

Pipelining nodes that pushes the contents of the named file stream to the next node in the pipeline.

Parameters
pathThe file stream from which it pushes items

Definition at line 389 of file file_stream.h.

A pipelining node that prints the items that are pushed to it.

Definition at line 76 of file stdio.h.

A node that allows peeking at the next item in the pipeline.

Definition at line 484 of file helpers.h.

Constructs a reverser node stored in external memory.

Reverses the stream and creates a phase boundary

Definition at line 370 of file reverse.h.

A pipelining node that pushes the integers it reads using scanf.

Definition at line 71 of file stdio.h.

A pipelining node that acts as a buffer for serializable items and creates a phase boundary.

Definition at line 499 of file serialization.h.

A pipelining node that reverses serializable items and creates a phase boundary.

Definition at line 493 of file serialization.h.

Enumeration Type Documentation

Type describing whether to maintain the order of items in parallel.

Enumerator
arbitrary_order 

Do not maintain order; push items as soon as a worker has processed them.

maintain_order 

Maintain order; push items in the same order that a single thread would have.

Definition at line 31 of file maintain_order_type.h.

31  {
34  arbitrary_order = false,
37  maintain_order = true
38 };
Do not maintain order; push items as soon as a worker has processed them.
Maintain order; push items in the same order that a single thread would have.

Function Documentation

template<typename T >
pipe_begin<factory<bits::ami_input_t, tpie::ami::stream<T> &> > tpie::pipelining::ami_input ( tpie::ami::stream< T > &  input)
inline

Pipelining nodes that pushes the contents of the given ami::stream to the next node in the pipeline.

Parameters
inputThe ami::stream from which it pushes items

Definition at line 142 of file ami_glue.h.

References input().

142  {
143  return factory<bits::ami_input_t, tpie::ami::stream<T> &>(input);
144 }
template<typename T >
pipe_begin<factory<bits::ami_input_stack_t, tpie::ami::stack<T> &> > tpie::pipelining::ami_input_stack ( tpie::ami::stack< T > &  input)
inline

Pipelining nodes that pushes the contents of the given ami::stack to the next node in the pipeline.

Parameters
inputThe ami::stack from which it pushes items

Definition at line 153 of file ami_glue.h.

References input().

153  {
154  return factory<bits::ami_input_stack_t, tpie::ami::stack<T> &>(input);
155 }
template<typename T >
pipe_end<termfactory<bits::ami_output_t<T>, tpie::ami::stream<T> &> > tpie::pipelining::ami_output ( tpie::ami::stream< T > &  fs)
inline

A pipelining node that writes the pushed items to an ami stream.

Parameters
fsThe stream that items should be written to

Definition at line 172 of file ami_glue.h.

172  {
173  return termfactory<bits::ami_output_t<T>, tpie::ami::stream<T> &>(fs);
174 }
template<typename T >
pullpipe_begin<termfactory<bits::pull_input_t<T>, tpie::ami::stack<T> &> > tpie::pipelining::ami_pull_input_stack ( tpie::ami::stack< T > &  fs)
inline

A pipelining pull-node that reads items from the given ami::stack.

Parameters
fsThe ami::stack from which it reads items.

Definition at line 163 of file ami_glue.h.

163  {
164  return termfactory<bits::pull_input_t<T>, tpie::ami::stack<T> &>(fs);
165 }
An implementation of an external-memory stack compatible with the old AMI interface.
Definition: stack.h:180
pipe_middle<factory<bits::ostream_logger_t, std::ostream &> > tpie::pipelining::cout_logger ( )
inline

A pipelining node that writes items to standard out and then pushes them to the next node.

Definition at line 477 of file helpers.h.

477  {
478  return {std::cout};
479 }
template<typename F >
pipe_middle<tempfactory<bits::exclude_lambda_t<F>, F> > tpie::pipelining::exclude_lambda ( const F &  f)
inline

Pipelining nodes that applies to given functor to items in the stream.

The functor should have a typedef named argument_type that is the type of the argument given to the call operator. It is required that the functor returns a pair. The first item should be a boolean indicating whether the item should be pushed to the next node. The second should be the value itself.

Parameters
fThe functor that should be applied to items

Definition at line 203 of file std_glue.h.

203  {
204  return tempfactory<bits::exclude_lambda_t<F>, F>(f);
205 }
template<typename F >
pipe_middle<tempfactory<bits::filter_t<F>, F> > tpie::pipelining::filter ( const F &  functor)

A pipelining node that keeps only elements where functor evaluates to true.

Parameters
functorThe filter to use

Definition at line 64 of file filter.h.

64  {
65  return tempfactory<bits::filter_t<F>, F >(functor);
66 }
template<typename fact_t >
pipe_middle<tempfactory<bits::fork_t<fact_t>, fact_t> > tpie::pipelining::fork ( pipe_end< fact_t >  to)

Create a fork pipe node.

Whenever an element e is push into the fork node, e is pushed to the destination and then to "to"

Definition at line 495 of file helpers.h.

495  {
496  return {std::move(to.factory)};
497 }
pipe_middle<factory<bits::Forwarder, std::vector<std::pair<std::string, any_noncopyable> > > > tpie::pipelining::forwarder ( std::vector< std::pair< std::string, any_noncopyable > >  items)
inline

A pipelining node that will forward values on prepare, and.

Definition at line 59 of file forwarder.h.

Referenced by forwarder().

59  {
60  return factory<bits::Forwarder, std::vector<std::pair<std::string, any_noncopyable> > >(std::move(items));
61 }
template<typename VT >
pipe_middle<factory<bits::Forwarder, std::vector<std::pair<std::string, any_noncopyable> > > > tpie::pipelining::forwarder ( std::string  name,
VT  value 
)

A pipelining node that will forward value on prepare, and.

Definition at line 68 of file forwarder.h.

References forwarder().

68  {
69  std::vector<std::pair<std::string, any_noncopyable> > v;
70  v.push_back(std::make_pair(name, any_noncopyable(value)));
71  return forwarder(std::move(v));
72 }
pipe_middle< factory< bits::Forwarder, std::vector< std::pair< std::string, any_noncopyable > > > > forwarder(std::vector< std::pair< std::string, any_noncopyable > > items)
A pipelining node that will forward values on prepare, and.
Definition: forwarder.h:59
template<typename T >
pipe_begin<factory<bits::input_t, file_stream<T> &, stream_options> > tpie::pipelining::input ( file_stream< T > &  fs,
stream_options  options = stream_options() 
)
inline

Pipelining nodes that pushes the contents of the given file stream to the next node in the pipeline.

Parameters
fsThe file stream from which it pushes items
optionsStream options

Definition at line 379 of file file_stream.h.

Referenced by ami_input(), and ami_input_stack().

380  {
381  return {fs, options};
382 }
template<typename T , typename A >
pipe_begin<tfactory<bits::input_vector_t, Args<T, A>, const std::vector<T, A> &> > tpie::pipelining::input_vector ( const std::vector< T, A > &  input)
inline

Pipelining nodes that pushes the contents of the given vector to the next node in the pipeline.

Parameters
inputThe vector from which it pushes items

Definition at line 153 of file std_glue.h.

153  {
154  return {input};
155 }
template<typename T >
pipe_middle<tempfactory<bits::item_type_t<T> > > tpie::pipelining::item_type ( )

Create item type defining identity pipe node.

Defines the item_type to be T. Whenever an element is push, it is immidiately pushed to the destination

Definition at line 654 of file helpers.h.

654  {
655  return tempfactory<bits::item_type_t<T> >();
656 }
template<typename F >
pipe_middle<tempfactory<bits::lambda_t<F>, F> > tpie::pipelining::lambda ( const F &  f)
inline

Pipelining nodes that applies to given functor to items in the stream.

The functor should have a typedef named argument_type that is the type of the argument given to the call operator.

Parameters
fThe functor that should be applied to items

Definition at line 189 of file std_glue.h.

189  {
190  return tempfactory<bits::lambda_t<F>, F>(f);
191 }
template<typename T >
pipe_middle<factory<bits::linear_t, T, T> > tpie::pipelining::linear ( factor,
term 
)
inline

A pipelining node that transforms the items by applying a linear function to them.

Parameters
factorthe factor that items should be multiplied by
termthe term is added after the item is multipled by the factor

Definition at line 86 of file numeric.h.

86  {
87  return factory<bits::linear_t, T, T>(factor, term);
88 }
template<typename F , typename = typename std::enable_if<bits::has_argument_type<F>::value>::type>
pipe_middle<tempfactory<bits::map_t<F>, F> > tpie::pipelining::map ( const F &  functor)

Pipelining nodes that applies to given functor to items in the stream.

Parameters
fThe functor that should be applied to items

Definition at line 143 of file map.h.

143  {
144  return tempfactory<bits::map_t<F>, F >(functor);
145 }
template<typename pull_t >
pipe_middle<factory<bits::merge_t<pull_t>::template type, pull_t> > tpie::pipelining::merge ( pullpipe_begin< pull_t >  with)
inline

A node that merges a pull pipeline into a push pipeline.

It pulls an items for each item pushed to it.

Definition at line 73 of file merge.h.

73  {
74  return {std::move(with.factory)};
75 }
template<typename T >
pipe_end<termfactory<bits::named_output_t<T>, std::string> > tpie::pipelining::named_output ( std::string  path)
inline

A pipelining node that writes the pushed items to a named file stream.

Parameters
pathThe path of where to write the firestream

Definition at line 440 of file file_stream.h.

440  {
441  return {std::move(path)};
442 }
template<typename T >
pullpipe_begin<termfactory<bits::named_pull_input_t<T>, std::string> > tpie::pipelining::named_pull_input ( std::string  name)
inline

A pipelining pull-node that reads items from the given file_stream.

Parameters
fsThe file stream from which it reads items.

Definition at line 421 of file file_stream.h.

421  {
422  return {std::move(name)};
423 }
template<typename T >
pipe_end<termfactory<bits::null_sink_t<T> > > tpie::pipelining::null_sink ( )
inline

Create a dummy end pipe node.

Whenever an element of type T is pushed to the null_sink it is disregarded

Definition at line 543 of file helpers.h.

543 {return termfactory<bits::null_sink_t<T> >();}
template<typename T >
pipe_end<termfactory<bits::output_t<T>, file_stream<T> &> > tpie::pipelining::output ( file_stream< T > &  fs)
inline

A pipelining node that writes the pushed items to a file stream.

Parameters
fsThe file stream that items should be written to

Definition at line 431 of file file_stream.h.

Referenced by output_vector().

431  {
432  return {fs};
433 }
template<typename T , typename A >
pipe_end<termfactory<bits::output_vector_t<T, A>, std::vector<T, A> &> > tpie::pipelining::output_vector ( std::vector< T, A > &  output)
inline

Pipelining node that pushes items to the given vector.

Parameters
outputThe vector to push items to

Definition at line 176 of file std_glue.h.

References output().

176  {
177  return termfactory<bits::output_vector_t<T, A>, std::vector<T, A> &>(output);
178 }
template<typename fact_t >
pipe_middle<parallel_bits::factory<fact_t> > tpie::pipelining::parallel ( pipe_middle< fact_t > &&  fact,
maintain_order_type  maintainOrder,
size_t  numJobs,
size_t  bufSize = 2048 
)

Runs a pipeline in multiple threads.

Parameters
maintainOrderWhether to make sure that items are processed and output in the order they are input.
numJobsThe number of threads to utilize for parallel execution.
bufSizeThe number of items to store in the buffer sent between threads.

Definition at line 42 of file pipes.h.

References arbitrary_order, and maintain_order.

Referenced by parallel().

42  {
43  parallel_bits::options opts;
44  switch (maintainOrder) {
45  case arbitrary_order:
46  opts.maintainOrder = false;
47  break;
48  case maintain_order:
49  opts.maintainOrder = true;
50  break;
51  }
52  opts.numJobs = numJobs;
53  opts.bufSize = bufSize;
54  return pipe_middle<parallel_bits::factory<fact_t> >
55  (parallel_bits::factory<fact_t>
56  (std::move(fact.factory), std::move(opts)));
57 }
Do not maintain order; push items as soon as a worker has processed them.
Maintain order; push items in the same order that a single thread would have.
template<typename fact_t >
pipe_middle<parallel_bits::factory<fact_t> > tpie::pipelining::parallel ( pipe_middle< fact_t > &&  fact,
maintain_order_type  maintainOrder = arbitrary_order 
)

Runs a pipeline in multiple threads, using the number of threads reported by tpie::default_worker_count.

Parameters
maintainOrderWhether to make sure that items are processed and output in the order they are input.

Definition at line 67 of file pipes.h.

References tpie::default_worker_count(), and parallel().

67  {
68  return parallel(std::move(fact), maintainOrder, default_worker_count());
69 }
pipe_middle< parallel_bits::factory< fact_t > > parallel(pipe_middle< fact_t > &&fact, maintain_order_type maintainOrder, size_t numJobs, size_t bufSize=2048)
Runs a pipeline in multiple threads.
Definition: pipes.h:42
memory_size_type default_worker_count()
Return the number of job threads initialized by the job framework in init_job().

A pipelining node that removes duplicate items and create a phase boundary.

Definition at line 103 of file uniq.h.

103  {
104  return bits::pair_factory<factory<bits::count_consecutive_t>, factory<bits::extract_first_t> >
105  (factory<bits::count_consecutive_t>(), factory<bits::extract_first_t>());
106 }
template<typename F >
pipe_middle<tempfactory<bits::preparer_t<F>, F> > tpie::pipelining::preparer ( const F &  functor)

Create preparer callback identity pipe node.

When prepare is called on the node the functor is called Whenever an element is pushed, it is immidiately pushed to the destination

Definition at line 632 of file helpers.h.

632  {
633  return tempfactory<bits::preparer_t<F>, F>(functor);
634 }
template<typename F >
pipe_middle<tempfactory<bits::propagater_t<F>, F> > tpie::pipelining::propagater ( const F &  functor)

Create propagate callback identity pipe node.

When propagate is called on the node the functor is called Whenever an element is pushed, it is immidiately pushed to the destination

Definition at line 643 of file helpers.h.

643  {
644  return tempfactory<bits::propagater_t<F>, F>(functor);
645 }
template<typename dest_fact_t >
pullpipe_middle<tfactory<bits::pull_fork_t, Args<dest_fact_t>, dest_fact_t> > tpie::pipelining::pull_fork ( dest_fact_t  dest_fact)

Create a pulling fork pipe node.

Whenever an element e is pulled from fork node, e is first pushed into the destination

Definition at line 507 of file helpers.h.

507  {
508  return {std::move(dest_fact)};
509 }
template<typename T >
pullpipe_begin<termfactory<bits::pull_input_t<T>, file_stream<T> &, stream_options> > tpie::pipelining::pull_input ( file_stream< T > &  fs,
stream_options  options = stream_options() 
)
inline

A pipelining pull-node that reads items from the given file_stream.

Parameters
fsThe file stream from which it reads items.
optionsStream options

Definition at line 397 of file file_stream.h.

399  {
400  return {fs, options};
401 }
template<typename IT >
pullpipe_begin<termfactory<bits::pull_input_iterator_t<IT>, IT, IT> > tpie::pipelining::pull_input_iterator ( IT  begin,
IT  end 
)

A pull-pipe that returns items in the range given by two iterators.

Parameters
beginThe iterator pointing to the first item
endThe iterator pointing to the end of the range

Definition at line 576 of file helpers.h.

576  {
577  return termfactory<bits::pull_input_iterator_t<IT>, IT, IT>(begin, end);
578 }
template<typename T , typename A >
pullpipe_begin<termfactory<bits::pull_input_vector_t<T, A>, const std::vector<T, A> &> > tpie::pipelining::pull_input_vector ( const std::vector< T, A > &  input)
inline

Pipelining nodes that pushes the contents of the given vector to the next node in the pipeline.

Parameters
inputThe vector from which it pushes items

Definition at line 165 of file std_glue.h.

165  {
166  return {input};
167 }
template<typename T >
pullpipe_end<factory<bits::pull_output_t, file_stream<T> &> > tpie::pipelining::pull_output ( file_stream< T > &  fs)
inline

A pull-pipe node that writes the pulled items to a file stream.

Parameters
fsThe file stream that items should be written to

Definition at line 450 of file file_stream.h.

450  {
451  return {fs};
452 }
template<typename IT >
pullpipe_end<tempfactory<bits::pull_output_iterator_t<IT>, IT> > tpie::pipelining::pull_output_iterator ( IT  to)

A pull-pipe node that writes its given items to the destination pointed to by the given iterator.

Parameters
toAn iterator pointing to the destination that items should be written to

Definition at line 621 of file helpers.h.

621  {
622  return tempfactory<bits::pull_output_iterator_t<IT>, IT>(to);
623 }
template<typename T >
pullpipe_begin<termfactory<bits::pull_reverse_input_t<T>, file_stream<T> &, stream_options> > tpie::pipelining::pull_reverse_input ( file_stream< T > &  fs,
stream_options  options = stream_options() 
)
inline

A pipelining pull-node that reads items in reverse order from the given file_stream.

Parameters
fsThe file stream from which it reads items.
optionsStream options

Definition at line 410 of file file_stream.h.

412  {
413  return {fs, options};
414 }
template<typename fact_t >
pipe_begin<tempfactory<bits::pull_source_t<fact_t>, fact_t> > tpie::pipelining::pull_source ( pullpipe_begin< fact_t >  from)

A node that pulls items from source and push them into dest.

Parameters
Thepull source, and the source forwards the number of items, "items"

Definition at line 665 of file helpers.h.

665  {
666  return {std::move(from.factory)};
667 }
template<typename T >
pullpipe_middle<tfactory<bits::pull_tee_t, Args<typename T::item_type>, T &> > tpie::pipelining::pull_tee ( T &  fs)
inline

A pull-pipe node that when pulled from will pull from its source, write its item to disk and then return the item.

Parameters
fsThe file stream that items should be written to

Definition at line 470 of file file_stream.h.

470  {
471  return {fs};
472 }
template<typename IT >
pipe_begin<tempfactory<bits::push_input_iterator_t<IT>, IT, IT> > tpie::pipelining::push_input_iterator ( IT  begin,
IT  end 
)

A pipelining node that pushes the items in the range given by two iterators.

Parameters
beginThe iterator pointing to the first item
endThe iterator pointing to the end of the range

Definition at line 587 of file helpers.h.

587  {
588  return tempfactory<bits::push_input_iterator_t<IT>, IT, IT>(begin, end);
589 }
template<typename IT >
pipe_end<termfactory<bits::push_output_iterator_t<IT>, IT> > tpie::pipelining::push_output_iterator ( IT  to)

A node that writes its given items to the destination pointed to by the given iterator.

Parameters
toAn iterator pointing to the destination that items should be written to

Definition at line 598 of file helpers.h.

598  {
599  return termfactory<bits::push_output_iterator_t<IT>, IT>(to);
600 }
pipe_begin<serialization_bits::input_factory> tpie::pipelining::serialization_input ( serialization_reader &  rd)
inline

A pipelining node that reads items from a serialization_reader.

Parameters
rdThe reader from which which items should be read

Definition at line 110 of file serialization.h.

110  {
111  return pipe_begin<serialization_bits::input_factory>(&rd);
112 }
template<typename T >
pipe_end<typename serialization_bits::output_factory<T>::type> tpie::pipelining::serialization_output ( serialization_writer &  wr)

A pipelining node that writes item to a serialization_writer.

Parameters
wrThe writer to which items should be written

Definition at line 120 of file serialization.h.

120  {
121  return typename serialization_bits::output_factory<T>::type(&wr);
122 }
pipe_middle<serialization_bits::default_pred_sort_factory> tpie::pipelining::serialization_sort ( )
inline

Pipelining sorter using std::less.

Definition at line 424 of file serialization_sort.h.

424  {
425  typedef serialization_bits::default_pred_sort_factory fact;
426  return pipe_middle<fact>(fact()).name("Sort");
427 }
template<typename pred_t >
pipe_middle<serialization_bits::sort_factory<pred_t> > tpie::pipelining::serialization_sort ( const pred_t &  p)

Pipelining sorter using the given predicate.

Definition at line 434 of file serialization_sort.h.

434  {
435  typedef serialization_bits::sort_factory<pred_t> fact;
436  return pipe_middle<fact>(fact(p)).name("Sort");
437 }
pipe_middle<bits::default_pred_sort_factory<default_store> > tpie::pipelining::sort ( )
inline

Pipelining sorter using std::less.

Definition at line 492 of file sort.h.

492  {
493  typedef bits::default_pred_sort_factory<default_store> fact;
494  return pipe_middle<fact>(fact(default_store())).name("Sort");
495 }
template<typename pred_t >
pipe_middle<bits::sort_factory<pred_t, default_store> > tpie::pipelining::sort ( const pred_t &  p)
inline

Pipelining sorter using the given predicate.

Definition at line 513 of file sort.h.

513  {
514  typedef bits::sort_factory<pred_t, default_store> fact;
515  return pipe_middle<fact>(fact(p, default_store())).name("Sort");
516 }
template<typename pred_t , typename store_t >
pipe_middle<bits::sort_factory<pred_t, store_t> > tpie::pipelining::sort ( const pred_t &  p,
store_t  store 
)
inline

A pipelining node that sorts large elements indirectly by using a storeand a given predicate.

Definition at line 524 of file sort.h.

524  {
525  typedef bits::sort_factory<pred_t, store_t> fact;
526  return pipe_middle<fact>(fact(p, store)).name("Sort");
527 }
template<typename store_t >
pipe_middle<bits::default_pred_sort_factory<store_t> > tpie::pipelining::store_sort ( store_t  store = store_t())
inline

A pipelining node that sorts large elements indirectly by using a store and std::less.

Definition at line 503 of file sort.h.

503  {
504  typedef bits::default_pred_sort_factory<store_t> fact;
505  return pipe_middle<fact>(fact(store)).name("Sort");
506 }
template<typename T >
pipe_middle<tfactory<bits::tee_t, Args<typename T::item_type>, T &> > tpie::pipelining::tee ( T &  fs)
inline

A pipelining node that writes the pushed to a file stream and then pushes the items to the next node.

Parameters
fsThe file stream that items should be written to

Definition at line 460 of file file_stream.h.

460  {
461  return {fs};
462 }
template<typename Item , typename IT >
pipe_end<termfactory<bits::push_output_iterator_t<IT, Item>, IT> > tpie::pipelining::typed_push_output_iterator ( IT  to)

A node that writes its given items to the destination pointed to by the given iterator.

Parameters
toAn iterator pointing to the destination that items should be written to
Template Parameters
ItemThe type of the pushed items

Definition at line 610 of file helpers.h.

610  {
611  return termfactory<bits::push_output_iterator_t<IT, Item>, IT>(to);
612 }
template<typename equal_t >
pipe_middle<tfactory<bits::unique_t, Args<equal_t>, equal_t> > tpie::pipelining::unique ( equal_t  equal)

Filter consecutive duplicates out.

When items are pushed in Whenever a pushed itme is same as the previous, it is dropped

Definition at line 676 of file helpers.h.

676  {
677  return {equal};
678 }
template<typename fact_t >
pipe_middle<tempfactory<bits::unzip_t<fact_t>, fact_t> > tpie::pipelining::unzip ( pipe_end< fact_t >  to)

Create unzip pipe node.

Whenever a std::pair<A,B>(a,b) is pushed to the unzip node, a is pushed to its destination, and then b is pushed to "to"

Definition at line 519 of file helpers.h.

519  {
520  return {std::move(to.factory)};
521 }
template<typename F >
pipe_middle<tempfactory<bits::visit_t<F>, F> > tpie::pipelining::visit ( const F &  functor)

A pipelining node that applies a functor to elements pushed to it.

Parameters
functorThe visitor to use

Definition at line 61 of file visit.h.

61  {
62  return tempfactory<bits::visit_t<F>, F >(functor);
63 }
template<typename T >
pullpipe_begin<termfactory<bits::zero_source_t<T> > > tpie::pipelining::zero_source ( )
inline

Create a dummy pull begin pipe node.

Whenever an element of type T is pushed to the null_sink it is disregarded

Definition at line 552 of file helpers.h.

552 {return termfactory<bits::zero_source_t<T> >();}
template<typename fact_t >
pipe_middle<tempfactory<bits::zip_t<fact_t>, fact_t> > tpie::pipelining::zip ( pullpipe_begin< fact_t >  from)

Create a zip pipe node.

Whenever an element a is pushed to the zip node, an element b is pulled from the "from" node, and std::make_pair(a,b) is pushed to the destination

Definition at line 532 of file helpers.h.

532  {
533  return {std::move(from.factory)};
534 }