Qore DataProvider Module Reference  1.0.2
DataProviderPipeline.qc.dox.h
1 // -*- mode: c++; indent-tabs-mode: nil -*-
3 
25 // assume local scope for variables, do not use "$" signs
26 // require type definitions everywhere
28 // enable all warnings
29 // allow weak references
30 
32 namespace DataProvider {
36 const PS_ABORTED = "ABORTED";
38 
40 const PS_RUNNING = "RUNNING";
41 
43 const PS_IDLE = "IDLE";
45 
49 
53 const DPE_SKIP_DATA = "DPE-SKIP-DATA";
55 
57 public struct PipelineInfo {
59  string name;
60 
63 
66 
68 
70  string status;
71 
74 
77 
79  bool bulk;
80 
83 
86 
88  float recs_per_sec;
89 };
90 
92 public struct PipelineOptionInfo {
94 
96  *code debug_log;
97 
99 
101  *code error_log;
102 
104 
106  *code info_log;
107 
109  /* it is called in the new pipeline thread with no arguments
110  */
112 
114  *string name;
115 };
116 
119 
120 public:
122  int id;
123 
125  Mutex lck;
126 
128  Counter cnt;
129 
131  Condition cond();
132 
134  int queue_waiting = 0;
135 
137  int data_waiting = 0;
138 
140  list<auto> queue;
141 
143  int size;
144 
146  int tid;
147 
149 
153  list<auto> elems();
154 
157 
159  constructor(DataProviderPipeline parent, Mutex lck, Counter cnt, int id, int size);
160 
161 
163  int getId();
164 
165 
167 
171  submit(auto record);
172 
173 
175  run(Counter run_cnt);
176 
177 
179 
182 
183 };
184 
186 
190 
191 public:
192 
193 
194 protected:
196  string name;
197 
199  hash<string, PipelineQueue> pmap;
200 
202  bool do_bulk = True;
203 
205  bool locked = False;
206 
208  Sequence seq(1);
209 
211  list<hash<ExceptionInfo>> error_list;
212 
215 
218 
220  int record_count = 0;
221 
223  *code info_log;
224 
226  *code error_log;
227 
229  *code debug_log;
230 
232  /* it is called in the new pipeline thread with no arguments
233  */
235 
237  Mutex lck();
238 
240  bool stop_flag;
241 
244 
246  Counter cnt();
247 
248 public:
249 
251 
255  constructor(*hash<PipelineOptionInfo> opts);
256 
257 
259 
261  copy();
262 
263 
265 
272 
273 
275  string getName();
276 
277 
279  bool hasQueue(int id);
280 
281 
283  bool stopping();
284 
285 
287  bool aborting();
288 
289 
292 
293 
295 
301 
302 
304 
314  append(int id, AbstractDataProcessor processor);
315 
316 
318 
328  int appendQueue(int id);
329 
330 
332  bool isProcessing();
333 
334 
336  submit(auto _data);
337 
338 
340  submitData(AbstractIterator i);
341 
342 
345 
346 
349 
350 
352 
355  reset();
356 
357 
359 
363 
364 
366 
374  abort(*bool ignore_exceptions);
375 
376 
378 
382  hash<PipelineInfo> getInfo();
383 
384 
386  reportError(PipelineQueue queue, hash<ExceptionInfo> ex);
387 
388 
390  logInfo(string fmt);
391 
392 
394  logError(string fmt);
395 
396 
398  logDebug(string fmt);
399 
400 
402 private:
404 public:
405 
406 
408 
410 private:
411  submitIntern(auto _data);
412 public:
413 
414 
416 
418 private:
419  submitDataIntern(auto _data);
420 public:
421 
422 
424 
426 private:
428 public:
429 
430 
432 
434 private:
436 public:
437 
438 
440 
442 private:
444 public:
445 
446 
448 
450 private:
452 public:
453 
454 
456 
458 private:
460 public:
461 
462 
464 private:
466 public:
467 
468 
470 private:
472 public:
473 
474 
476 private:
478 public:
479 
480 };
481 };
DataProvider::DPE_SKIP_DATA
const DPE_SKIP_DATA
Skips processing a record.
Definition: DataProviderPipeline.qc.dox.h:53
DataProvider::PipelineQueue::lck
Mutex lck
Parent lock.
Definition: DataProviderPipeline.qc.dox.h:125
DataProvider::DataProviderPipeline::logError
logError(string fmt)
Logs to the error log, if set.
DataProvider::DataProviderPipeline::start_time
date start_time
run start time
Definition: DataProviderPipeline.qc.dox.h:214
DataProvider::PipelineOptionInfo::name
*string name
the name of the pipeline for logging purposes; if this key is not included, a generic name will be ge...
Definition: DataProviderPipeline.qc.dox.h:114
DataProvider::PipelineOptionInfo::debug_log
*code debug_log
a closure or call reference for debug logging
Definition: DataProviderPipeline.qc.dox.h:96
DataProvider::DataProviderPipeline::pmap
hash< string, PipelineQueue > pmap
Hash of queues keyed by queue ID.
Definition: DataProviderPipeline.qc.dox.h:199
DataProvider::DataProviderPipeline::locked
bool locked
Locked flag.
Definition: DataProviderPipeline.qc.dox.h:205
DataProvider::DataProviderPipeline::name
string name
A descriptive name for logging purposes.
Definition: DataProviderPipeline.qc.dox.h:196
DataProvider::DataProviderPipeline::copyPipeline
PipelineQueue copyPipeline(PipelineQueue old_queue)
Called by the copy constructor to copy the queues.
DataProvider::DataProviderPipeline::append
append(int id, AbstractDataProcessor processor)
Appends a data processor to a pipeline.
DataProvider::DataProviderPipeline::submit
submit(auto _data)
Submits data for processing.
DataProvider::DataProviderPipeline::stop_flag
bool stop_flag
Stop flag.
Definition: DataProviderPipeline.qc.dox.h:240
DataProvider::PipelineQueue::submit
submit(auto record)
Submits data for processing.
DataProvider::DataProviderPipeline::reset
reset()
Resets the pipeline.
DataProvider::PipelineQueue::getId
int getId()
Returns the pipeline ID.
DataProvider::PipelineInfo
Pipeline info.
Definition: DataProviderPipeline.qc.dox.h:57
DataProvider::PipelineQueue::run
run(Counter run_cnt)
Processing thread.
DataProvider::PipelineQueue::data_waiting
int data_waiting
Number of threads waiting on data.
Definition: DataProviderPipeline.qc.dox.h:137
DataProvider::PipelineInfo::start_time
*date start_time
Start of processing.
Definition: DataProviderPipeline.qc.dox.h:62
DataProvider::DataProviderPipeline::error_log
*code error_log
Error log closure; takes a single format string and then arguments for format placeholders.
Definition: DataProviderPipeline.qc.dox.h:226
DataProvider::DataProviderPipeline::debug_log
*code debug_log
Debug log closure; takes a single format string and then arguments for format placeholders.
Definition: DataProviderPipeline.qc.dox.h:229
DataProvider::PipelineInfo::status
string status
Pipeline status.
Definition: DataProviderPipeline.qc.dox.h:70
True
const True
DataProvider::DataProviderPipeline::append
append(AbstractDataProcessor processor)
Appends a data processor to the default pipeline.
DataProvider::DataProviderPipeline::getName
string getName()
Returns the pipeline name.
DataProvider::PipelineOptionInfo::error_log
*code error_log
a closure or call reference for error logging
Definition: DataProviderPipeline.qc.dox.h:101
DataProvider::PipelineQueue::cond
Condition cond()
Queue condition variable.
DataProvider::DataProviderPipeline::appendQueue
int appendQueue(int id)
Appends a new queue to an existing pipeline and returns the new queue ID.
DataProvider
Qore AbstractDataField class definition.
Definition: AbstractDataField.qc.dox.h:32
DataProvider::AbstractDataProcessor
Defines an abstract class for accepting data and outputting optionally transformed or filtered data.
Definition: AbstractDataProcessor.qc.dox.h:33
DataProvider::PS_RUNNING
const PS_RUNNING
Pipeline status: RUNNING.
Definition: DataProviderPipeline.qc.dox.h:40
DataProvider::DataProviderPipeline::isProcessing
bool isProcessing()
Returns True if the pipeline is processing data.
DataProvider::PipelineOptionInfo::thread_callback
*code thread_callback
a closure or call reference for setting thread-local data in new pipeline threads
Definition: DataProviderPipeline.qc.dox.h:111
DataProvider::DataProviderPipeline::do_bulk
bool do_bulk
Bulk flag.
Definition: DataProviderPipeline.qc.dox.h:202
DataProvider::PipelineInfo::record_count
int record_count
Number of input records submitted.
Definition: DataProviderPipeline.qc.dox.h:73
DataProvider::DataProviderPipeline::aborting
bool aborting()
Returns True if the object is aborting.
DataProvider::PipelineOptionInfo::info_log
*code info_log
a closure or call reference for info logging
Definition: DataProviderPipeline.qc.dox.h:106
DataProvider::PipelineQueue::constructor
constructor(DataProviderPipeline parent, Mutex lck, Counter cnt, int id, int size)
Creates the object.
DataProvider::PipelineInfo::num_queues
int num_queues
Number of pipeline queues.
Definition: DataProviderPipeline.qc.dox.h:76
DataProvider::DataProviderPipeline::submit
submit(AbstractDataProviderBulkRecordInterface i)
Submits data for processing.
DataProvider::PipelineQueue::parent
DataProviderPipeline parent
Parent object.
Definition: DataProviderPipeline.qc.dox.h:156
DataProvider::DataProviderPipeline::stop_time
date stop_time
run stop time (set in waitDone())
Definition: DataProviderPipeline.qc.dox.h:217
DataProvider::DataProviderPipeline::abort
abort(*bool ignore_exceptions)
Aborts execution of a pipeline in progress.
DataProvider::PipelineQueue::elems
list< auto > elems()
Pipeline elements.
DataProvider::AbstractDataProviderRecordIterator
Defines the abstract class for data provider iterators; the destructor releases the iterator.
Definition: AbstractDataProviderRecordIterator.qc.dox.h:33
DataProvider::DataProviderPipeline::abort_flag
bool abort_flag
Abort flag.
Definition: DataProviderPipeline.qc.dox.h:243
DataProvider::PipelineOptionInfo
Pipeline option info.
Definition: DataProviderPipeline.qc.dox.h:92
DataProvider::DataProviderPipeline::copy
copy()
Copy constructor; creates an empty pipeline with the same configuration as the original.
DataProvider::PipelineQueue
Pipeline element.
Definition: DataProviderPipeline.qc.dox.h:118
DataProvider::DataProviderPipeline::error_list
list< hash< ExceptionInfo > > error_list
list of exceptions in pipelines
Definition: DataProviderPipeline.qc.dox.h:211
DataProvider::DataProviderPipeline::destructor
destructor()
Destroys the object.
DataProvider::DataProviderPipeline::thread_callback
*code thread_callback
a closure or call reference for setting thread-local data in new pipeline threads
Definition: DataProviderPipeline.qc.dox.h:234
DataProvider::PipelineQueue::queue_waiting
int queue_waiting
Number of threads waiting data to be removed from the queue.
Definition: DataProviderPipeline.qc.dox.h:134
DataProvider::DataProviderPipeline::waitDone
waitDone()
Waits for all queues in all pipelines to have processed remaining data.
DataProvider::AbstractDataProviderBulkRecordInterface
Defines the abstract class for data provider iterators; the destructor releases the iterator.
Definition: AbstractDataProviderBulkRecordInterface.qc.dox.h:33
DataProvider::DataProviderPipeline::submitData
submitData(AbstractIterator i)
Submits data for processing.
DataProvider::DataProviderPipeline::submitIntern
submitIntern(auto _data)
Submits data for processing.
DataProvider::PipelineInfo::stop_time
*date stop_time
Stop time for processing.
Definition: DataProviderPipeline.qc.dox.h:65
DataProvider::PipelineQueue::waitDone
waitDone()
Wait for the queue to be empty, then wait for all terminating pipelines to be empty.
DataProvider::PS_IDLE
const PS_IDLE
Pipeline status: IDLE.
Definition: DataProviderPipeline.qc.dox.h:43
DataProvider::PipelineQueue::size
int size
Maximum queue size.
Definition: DataProviderPipeline.qc.dox.h:143
DataProvider::PipelineInfo::duration_secs
float duration_secs
Total time processing end to end as a floating-point value in durationSecondsFloat.
Definition: DataProviderPipeline.qc.dox.h:85
DataProvider::DataProviderPipeline::reportError
reportError(PipelineQueue queue, hash< ExceptionInfo > ex)
Called from a pipeline queue object to report a fatal error durring processing.
DataProvider::DataProviderPipeline::checkLockedIntern
checkLockedIntern()
Throws an exception if the pipeline is locked.
DataProvider::PipelineQueue::id
int id
Pipeline ID.
Definition: DataProviderPipeline.qc.dox.h:122
DataProvider::DataProviderPipeline::info_log
*code info_log
Info log closure; takes a single format string and then arguments for format placeholders.
Definition: DataProviderPipeline.qc.dox.h:223
DataProvider::PipelineInfo::recs_per_sec
float recs_per_sec
Records processed per second end to end.
Definition: DataProviderPipeline.qc.dox.h:88
DataProvider::DataProviderPipeline::throwPipelineException
throwPipelineException()
Throws an exception if errors occured in background pipeline processing.
DataProvider::DataProviderPipeline::checkUpdatePipelineIntern
checkUpdatePipelineIntern(int id)
Check if the given queue exists.
False
const False
DataProvider::DataProviderPipeline::stopInternUnlocked
stopInternUnlocked()
Stops all background pipeline queues; lock must be held.
DataProvider::PipelineQueue::tid
int tid
TID of the background thread.
Definition: DataProviderPipeline.qc.dox.h:146
DataProvider::DataProviderPipeline::submit
submit(AbstractDataProviderRecordIterator i)
Submits data for processing.
DataProvider::DataProviderPipeline::logDebug
logDebug(string fmt)
Logs to the debug log, if set.
DataProvider::DataProviderPipeline::checkSubmitIntern
checkSubmitIntern()
Throws an exception if the pipeline cannot be used; locks the pipeline for changes otherwise.
DataProvider::DataProviderPipeline::record_count
int record_count
Record count.
Definition: DataProviderPipeline.qc.dox.h:220
DataProvider::DataProviderPipeline::logInfo
logInfo(string fmt)
Logs to the info log, if set.
DataProvider::DataProviderPipeline::constructor
constructor(*hash< PipelineOptionInfo > opts)
Creates the object with the given options.
DataProvider::DataProviderPipeline::hasQueue
bool hasQueue(int id)
Returns True if the given queue exists, False if not.
DataProvider::DataProviderPipeline::getInfo
hash< PipelineInfo > getInfo()
Returns pipeline info.
DataProvider::DataProviderPipeline::seq
Sequence seq(1)
Pipeline ID sequence generator.
DataProvider::PipelineQueue::queue
list< auto > queue
Data queue.
Definition: DataProviderPipeline.qc.dox.h:140
DataProvider::DataProviderPipeline::cnt
Counter cnt()
Thread counter.
DataProvider::PipelineInfo::duration
date duration
Total time processing end to end.
Definition: DataProviderPipeline.qc.dox.h:82
date
date date(date dt)
DataProvider::DataProviderPipeline::submitDataIntern
submitDataIntern(auto _data)
Submits a single record for processing.
DataProvider::DataProviderPipeline
Defines a class for passing data through record processors.
Definition: DataProviderPipeline.qc.dox.h:189
DataProvider::PipelineInfo::name
string name
The name of the pipeline.
Definition: DataProviderPipeline.qc.dox.h:59
DataProvider::DataProviderPipeline::submitBulkIntern
submitBulkIntern(AbstractDataProviderBulkRecordInterface i)
Submits bulk data for processing.
DataProvider::PipelineQueue::cnt
Counter cnt
Parent counter.
Definition: DataProviderPipeline.qc.dox.h:128
DataProvider::PS_ABORTED
const PS_ABORTED
Pipeline status: ABORTED.
Definition: DataProviderPipeline.qc.dox.h:37
DataProvider::DataProviderPipeline::lck
Mutex lck()
Atomic lock.
DataProvider::DataProviderPipeline::stopIntern
stopIntern()
Stops all background pipeline queues.
DataProvider::DataProviderPipeline::stopping
bool stopping()
Returns True if the object is stopping.
DataProvider::DataProviderPipeline::registerThread
registerThread(PipelineQueue queue)
Registers a new thread.
DataProvider::PipelineInfo::bulk
bool bulk
Flag that indicates if the pipeline is capable of bulk record processing.
Definition: DataProviderPipeline.qc.dox.h:79
DataProvider::DataProviderPipeline::resetIntern
resetIntern()
Resets the pipeline.