UltraScan III
us_data_model.cpp
Go to the documentation of this file.
1 
3 #include "us_data_model.h"
4 #include "us_data_process.h"
5 #include "us_data_tree.h"
6 #include "us_util.h"
7 #include "us_settings.h"
8 #include "us_model.h"
9 #include "us_noise.h"
10 #include "us_editor.h"
11 
12 #define timeFmt QString("hh:mm:ss")
13 #define nowTime() "T="+QDateTime::currentDateTime().toString(timeFmt)
14 
15 // Scan the database and local disk for R/E/M/N data sets
16 US_DataModel::US_DataModel( QWidget* parwidg /*=0*/ )
17 {
18  parentw = parwidg; // parent (main manage_data) widget
19 
20  ddescs .clear(); // db descriptions
21  ldescs .clear(); // local descriptions
22  adescs .clear(); // all descriptions
23  chgrows.clear(); // changed rows
24 
25  dbg_level = US_Settings::us_debug();
26 }
27 
28 // Set database related pointers
29 void US_DataModel::setDatabase( US_DB2* a_db )
30 {
31  db = a_db; // pointer to opened db connection
32  invID = QString::number( US_Settings::us_inv_ID() );
33 DbgLv(1) << "DMod:setDB: invID" << invID;
34 }
35 
36 // Set progress bar related pointers
37 void US_DataModel::setProgress( QProgressBar* a_progr, QLabel* a_lbstat )
38 {
39  progress = a_progr; // pointer to progress bar
40  lb_status = a_lbstat; // pointer to status label
41 }
42 
43 // set sibling classes pointers
44 void US_DataModel::setSiblings( QObject* a_proc, QObject* a_tree )
45 {
46  ob_process = a_proc; // pointer to sister DataProcess object
47  ob_tree = a_tree; // pointer to sister DataTree object
48 }
49 
50 // Get database pointer
51 US_DB2* US_DataModel::dbase()
52 {
53  return db;
54 }
55 
56 // Get progress bar pointer
57 QProgressBar* US_DataModel::progrBar()
58 {
59  return progress;
60 }
61 
62 // Get status label pointer
63 QLabel* US_DataModel::statlab()
64 {
65  return lb_status;
66 }
67 
68 // get us_data_process object pointer
69 QObject* US_DataModel::procobj()
70 {
71  return ob_process;
72 }
73 
74 // get us_data_tree object pointer
75 QObject* US_DataModel::treeobj()
76 {
77  return ob_tree;
78 }
79 
80 // Scan the database and local for run IDs then return to caller
81 void US_DataModel::getRunIDs( QStringList& runIDs )
82 {
83  runIDs.clear();
84 
85  // Get a list of runIDs from the database
86  QStringList query;
87  query << "get_experiment_desc" << invID;
88  db->query( query );
89 
90  while ( db->next() )
91  {
92  QString runID = db->value( 1 ).toString();
93 
94  if ( ! runIDs.contains( runID ) )
95  runIDs << runID;
96  }
97 DbgLv(1) << "gRI: db runs" << runIDs.size();
98 
99  // Add any local runIDs not already represented
100 // QString rdir = US_Settings::resultDir();
101  QString rdir = US_Settings::resultDir() + "/";
102  QStringList aucdirs = QDir( rdir )
103  .entryList( QDir::AllDirs | QDir::NoDotAndDotDot, QDir::Name );
104 // rdir += "/";
105  QStringList aucfilt;
106  aucfilt << "*.auc";
107 DbgLv(1) << "gRI: aucdirs" << aucdirs.size();
108 
109  for ( int ii = 0; ii < aucdirs.size(); ii++ )
110  { // Loop thru potential data directories; add any new that have AUC content
111  QString aucdir = aucdirs.at( ii );
112  QString subdir = rdir + aucdir;
113  QStringList aucfiles = QDir( subdir )
114  .entryList( aucfilt, QDir::Files, QDir::Name );
115  int naucf = aucfiles.size();
116  QString runID = aucdir.section( ".", 0, 0 );
117 
118  if ( naucf > 0 && ! runIDs.contains( runID ) )
119  runIDs << runID;
120  }
121 DbgLv(1) << "gRI: db+local runs" << runIDs.size();
122 
123  runIDs.sort();
124 }
125 
126 // Scan the database and local for triples in a runID then return to caller
127 void US_DataModel::getTriples( QStringList& triples, QString runID )
128 {
129  triples.clear();
130 
131  // Browse raw data with matching runID to accumulate triples
132  QStringList query;
133  query << "get_experiment_info_by_runID" << runID << invID;
134  db->query( query );
135  db->next();
136  QString expID = db->value( 1 ).toString();
137 
138  query.clear();
139  query << "get_rawDataIDs" << expID;
140  db->query( query );
141 DbgLv(1) << "gTr: runID" << runID << "expID" << expID;
142 
143  while ( db->next() )
144  {
145  QString fname = db->value( 2 ).toString().section( "/", -1, -1 );
146  QString rrID = fname.section( ".", 0, 0 );
147 //DbgLv(1) << "gTr: rrID" << rrID << "fname" << fname;
148 
149  if ( rrID == runID )
150  { // Matching runID, so add triple (if need be)
151  QString triple = fname.section( ".", -4, -2 );
152 
153  if ( ! triples.contains( triple ) )
154  triples << triple;
155  }
156  }
157 DbgLv(1) << "gTr: db triples" << triples.size();
158 
159  // Add any local triples not already represented
160  QString aucdir = US_Settings::resultDir() + "/" + runID + "/";
161 DbgLv(1) << "gTr: aucdir" << aucdir;
162 
163  QStringList aucfilt;
164  aucfilt << runID + "*.auc";
165  QStringList aucfiles = QDir( aucdir )
166  .entryList( aucfilt, QDir::Files, QDir::Name );
167  int naucf = aucfiles.size();
168 DbgLv(1) << "gTr: naucf" << naucf;
169 
170  for ( int ii = 0; ii < naucf; ii++ )
171  { // Loop thru files to add any new triples found
172  QString fname = aucfiles.at( ii );
173  QString triple = fname.section( ".", -4, -2 );
174 
175  if ( ! triples.contains( triple ) )
176  triples << triple;
177  }
178 DbgLv(1) << "gTr: db+local triples" << triples.size();
179 
180  triples.sort();
181 }
182 
183 // Set run and triple filters
184 void US_DataModel::setFilters( QString a_runf, QString a_tripf, QString a_srcf )
185 {
186  if ( filt_run != a_runf ||
187  filt_triple != a_tripf ||
188  filt_source != a_srcf )
189  chgrows.clear(); // Reset changed rows if any filters changed
190 
191  filt_run = a_runf; // Filter string for runID
192  filt_triple = a_tripf; // Filter string for triple
193  filt_source = a_srcf; // Filter string for source (DB/local)
194 }
195 
196 // Scan the database and local disk for R/E/M/N data sets
197 void US_DataModel::scan_data()
198 {
199 DbgLv(1) << "ScnD: start scan " << nowTime();
200  scan_dbase( ); // Read db to build db descriptions
201 DbgLv(1) << "ScnD: DB scan done " << nowTime();
202 
203  sort_descs( ddescs ); // Sort db descriptions
204 DbgLv(1) << "ScnD: DB sort done " << nowTime();
205 
206  scan_local( ); // Read files to build local descriptions
207 DbgLv(1) << "ScnD: Lcl scan done" << nowTime();
208 
209  sort_descs( ldescs ); // Sort local descriptions
210 DbgLv(1) << "ScnD: Lcl sort done" << nowTime();
211 
212  merge_dblocal(); // Merge database and local descriptions
213 DbgLv(1) << "ScnD: Merge done " << nowTime();
214 
215  if ( filt_source.startsWith( "Exclude" ) )
216  {
217  exclude_trees(); // Exclude DB-Only or Local-Only trees
218  }
219 }
220 
221 // Get data description object at specified row
222 US_DataModel::DataDesc US_DataModel::row_datadesc( int irow )
223 {
224  return adescs.at( irow );
225 }
226 
227 // Get current data description object
228 US_DataModel::DataDesc US_DataModel::current_datadesc( )
229 {
230  return cdesc;
231 }
232 
233 // Change data description object at a specified row
234 void US_DataModel::change_datadesc( DataDesc ddesc, int row )
235 {
236  adescs[ row ] = ddesc;
237  cdesc = ddesc;
238  chgrows << row;
239 }
240 
241 // Set current data description object
242 void US_DataModel::setCurrent( int irow )
243 {
244  cdesc = adescs.at( irow );
245 }
246 
247 // get count of total data records
248 int US_DataModel::recCount()
249 {
250  return adescs.size();
251 }
252 
253 // get count of DB data records
254 int US_DataModel::recCountDB()
255 {
256  return ddescs.size();
257 }
258 
259 // get count of local data records
260 int US_DataModel::recCountLoc()
261 {
262  return ldescs.size();
263 }
264 
265 // scan the database for R/E/M/N data sets
266 void US_DataModel::scan_dbase( )
267 {
268  const int max_qrec = 200;
269  QStringList rawIDs;
270  QStringList edtIDs;
271  QStringList modIDs;
272  QStringList noiIDs;
273  QStringList modDescs;
274  QStringList query;
275  QMap< QString, int > edtMap;
276  QMap< QString, QString > rawGUIDs;
277  QString dmyGUID = "00000000-0000-0000-0000-000000000000";
278  QString recID;
279  QString rawGUID;
280  QString contents;
281  int irecID;
282  int istep = 0;
283  int nstep = 1;
284  progress->setMaximum( nstep );
285  progress->setValue ( istep );
286 
287  if ( ! filt_source.isEmpty() && filt_source == "Local Only" )
288  { // If source filter is "Local-only", skip DB scan
289  ddescs.clear();
290  return;
291  }
292 
293  if ( chgrows.size() > 0 )
294  { // If changes since last scan, just modify existing db descriptions
295  review_dbase();
296  return;
297  }
298 
299  lb_status->setText( tr( "Reading DataBase Data..." ) );
300  qApp->processEvents();
301  ddescs.clear();
302 
303  if ( dbg_level > 1 )
304  {
305  query.clear();
306  query << "get_experiment_desc" << invID;
307  db->query( query );
308  QStringList expIDs;
309 
310  while ( db->next() )
311  {
312  QString expID = db->value( 0 ).toString();
313  QString runID = db->value( 1 ).toString();
314  QString etype = db->value( 2 ).toString();
315  expIDs << expID;
316 DbgLv(2) << " expID runID type" << expID << runID << etype;
317  }
318 
319  lb_status->setText( tr( "Reading Experiments" ) );
320 DbgLv(2) << " expID expGUID runID label comment date";
321  qApp->processEvents();
322 
323  for ( int ii = 0; ii < expIDs.size(); ii++ )
324  {
325  QString expID = expIDs[ ii ];
326  query.clear();
327  query << "get_experiment_info" << expID;
328  db->query( query );
329  db->next();
330  QString expGUID = db->value( 0 ).toString();
331  QString runID = db->value( 2 ).toString();
332  QString label = db->value( 9 ).toString();
333  QString comment = db->value( 10 ).toString();
334  QString date = US_Util::toUTCDatetimeText( db->value( 12 )
335  .toDateTime().toString( Qt::ISODate ), true );
336 DbgLv(2) << " " << expID << expGUID << runID << label << comment << date;
337  }
338  }
339 QDateTime basetime=QDateTime::currentDateTime();
340 
341  // Count raws, edits, models, noises
342  rawIDs .clear();
343  edtIDs .clear();
344  modIDs .clear();
345  noiIDs .clear();
346  bool rfilt = ( ! filt_run .isEmpty() && filt_run != "ALL" );
347  bool tfilt = ( ! filt_triple.isEmpty() && filt_triple != "ALL" );
348 
349  QString expID;
350  QString expGUID;
351  int nraws = 0;
352  int nedts = 0;
353  int nmods = 0;
354  int nnois = 0;
355 
356  if ( rfilt )
357  { // Count records when run/triple filtering
358 DbgLv(1) << "BrDb: filt'd Count start" << nowTime();
359  query.clear();
360  query << "get_experiment_info_by_runID" << filt_run << invID;
361  db->query( query );
362  db->next();
363  expID = db->value( 1 ).toString();
364  expGUID = db->value( 2 ).toString();
365 
366  query.clear();
367  query << "count_rawData_by_experiment" << expID;
368  nraws = db->functionQuery( query );
369 DbgLv(1) << "BrDb: nraws" << nraws;
370  rawIDs .reserve( nraws );
371 
372 DbgLv(1) << "BrDb: Count raws" << nowTime();
373  query.clear();
374  query << "get_rawDataIDs" << expID;
375  db->query( query );
376  nraws = 0;
377  while ( db->next() )
378  {
379  QString rawID = db->value( 0 ).toString();
380  QString filename = db->value( 2 ).toString().replace( "\\", "/" );
381  QString filebase = filename.section( "/", -1, -1 );
382  QString triple = filebase.section( ".", -4, -2 );
383  if ( tfilt && triple != filt_triple ) continue;
384  rawIDs << rawID;
385  nraws++;
386  }
387 DbgLv(1) << "BrDb: nraws" << nraws;
388 
389 DbgLv(1) << "BrDb: Count edits" << nowTime();
390  if ( nraws < max_qrec )
391  {
392  for ( int ii = 0; ii < nraws; ii++ )
393  {
394  QString rawID = rawIDs[ ii ];
395  query.clear();
396  query << "get_editedDataIDs" << rawID;
397  db->query( query );
398  while ( db->next() )
399  {
400  QString edtID = db->value( 0 ).toString();
401  QString filename = db->value( 2 ).toString()
402  .replace( "\\", "/" );
403  QString filebase = filename.section( "/", -1, -1 );
404  QString triple = filebase.section( ".", -4, -2 );
405  if ( tfilt && triple != filt_triple ) continue;
406  edtIDs << edtID;
407  nedts++;
408  }
409  }
410  }
411 
412  else
413  {
414  query.clear();
415  query << "all_editedDataIDs" << invID;
416  db->query( query );
417  while ( db->next() )
418  {
419  QString edtID = db->value( 0 ).toString();
420  QString expIDed = db->value( 4 ).toString();
421  if ( expIDed != expID ) continue;
422  QString filename = db->value( 2 ).toString().replace( "\\", "/" );
423  QString filebase = filename.section( "/", -1, -1 );
424  QString triple = filebase.section( ".", -4, -2 );
425  if ( tfilt && triple != filt_triple ) continue;
426  edtIDs << edtID;
427  nedts++;
428  }
429  }
430 DbgLv(1) << "BrDb: nedts" << nedts;
431 
432 DbgLv(1) << "BrDb: Count models,noises" << nowTime();
433  if ( nedts < max_qrec )
434  {
435  for ( int ii = 0; ii < nedts; ii++ )
436  {
437  QString edtID = edtIDs[ ii ];
438  query.clear();
439  query << "get_model_desc_by_editID" << invID << edtID;
440  db->query( query );
441  while ( db->next() )
442  {
443  QString modID = db->value( 0 ).toString();
444  modIDs << modID;
445  nmods++;
446  }
447  query.clear();
448  query << "get_noise_desc_by_editID" << invID << edtID;
449  db->query( query );
450  while ( db->next() )
451  {
452  QString noiID = db->value( 0 ).toString();
453  noiIDs << noiID;
454  nnois++;
455  }
456  }
457  }
458 
459  else
460  {
461  query.clear();
462  query << "get_model_desc" << invID;
463  db->query( query );
464  while ( db->next() )
465  {
466  QString modID = db->value( 0 ).toString();
467  QString edtID = db->value( 6 ).toString();
468  if ( edtIDs.contains( edtID ) )
469  {
470  modIDs << modID;
471  nmods++;
472  }
473  }
474  query.clear();
475  query << "get_noise_desc" << invID;
476  db->query( query );
477  while ( db->next() )
478  {
479  QString noiID = db->value( 0 ).toString();
480  QString edtID = db->value( 2 ).toString();
481  if ( edtIDs.contains( edtID ) )
482  {
483  noiIDs << noiID;
484  nnois++;
485  }
486  }
487  }
488 DbgLv(1) << "BrDb: nmods" << nmods << "nnois" << nnois;
489  }
490 
491  else
492  { // Count records when not run/triple filtering
493  query.clear();
494  query << "count_rawData" << invID;
495  nraws = db->functionQuery( query );
496 DbgLv(1) << "BrDb: nraws" << nraws;
497 
498  query.clear();
499  query << "count_editedData" << invID;
500  nedts = db->functionQuery( query );
501 DbgLv(1) << "BrDb: nedts" << nedts;
502 
503  query.clear();
504  query << "count_models" << invID;
505  nmods = db->functionQuery( query );
506 DbgLv(1) << "BrDb: nmods" << nmods;
507 
508  query.clear();
509  query << "count_noise" << invID;
510  nnois = db->functionQuery( query );
511 DbgLv(1) << "BrDb: nnois" << nnois;
512  }
513 
514  nstep = nraws + nedts + nmods + nnois;
515  int incre = nraws + nedts;
516 DbgLv(1) << "BrDb: nstep" << nstep << "incre" << incre;
517  incre = qMax( incre, 1 );
518  incre = ( nmods + nnois + incre - 1 ) / ( incre * 4 );
519  incre = qMax( incre, 1 );
520 DbgLv(1) << "BrDb: incre" << incre;
521  nstep += ( nraws + nedts ) * ( incre - 1 );
522  nstep = qMax( nstep, 1 );
523  istep = 0;
524 DbgLv(1) << "BrDb: nstep" << nstep;
525 //nstep=(nstep<1)?1000:nstep;
526  progress->setMaximum( nstep );
527  progress->setValue ( istep );
528  qApp->processEvents();
529 DbgLv(1) << "BrDb: # steps raws edts mods nois" << nstep << nraws << nedts
530  << nmods << nnois << "incre" << incre;
531 DbgLv(1) << "BrDb: count time:"
532  << basetime.msecsTo(QDateTime::currentDateTime())/1000.0;
533 
534  if ( !rfilt )
535  {
536  rawIDs .reserve( nraws );
537  edtIDs .reserve( nedts );
538  modIDs .reserve( nmods );
539  noiIDs .reserve( nnois );
540  }
541 
542  modDescs.reserve( nmods );
543 
544  // get raw data IDs
545  lb_status->setText( tr( "Reading Raws" ) );
546  qApp->processEvents();
547  int nqry = rfilt ? rawIDs.size() : qMin( nraws, 1 );
548  bool rfilt_q = rfilt && ( nqry < max_qrec );
549  nqry = rfilt_q ? nqry : qMin( nraws, 1 );
550 
551 DbgLv(1) << "BrDb: Query Raws" << nowTime() << "nqry" << nqry;
552  for ( int jq = 0; jq < nqry; jq++ )
553  {
554  QString rawID;
555  query.clear();
556  if ( rfilt_q )
557  {
558  rawID = rawIDs[ jq ];
559  query << "get_rawData" << rawID;
560  }
561  else
562  {
563  query << "all_rawDataIDs" << invID;
564  }
565  db->query( query );
566 
567  while ( db->next() )
568  { // Read Raw records
569  recID = db->value( 0 ).toString();
570  QString label = db->value( 1 ).toString();
571  QString filename = db->value( 2 ).toString().replace( "\\", "/" );
572  QString filebase = filename.section( "/", -1, -1 );
573  QString runID = filebase.section( ".", 0, 0 );
574  QString triple = filebase.section( ".", -4, -2 );
575 
576  if ( rfilt && runID != filt_run ) continue;
577  if ( tfilt && triple != filt_triple ) continue;
578 
579  QString experID;
580  QString date;
581  QString cksum;
582  QString recsize;
583  QString comment;
584 
585  if ( rfilt_q )
586  {
587  recID = rawID;
588  experID = db->value( 4 ).toString();
589  date = US_Util::toUTCDatetimeText( db->value( 7 )
590  .toDateTime().toString( Qt::ISODate ), true );
591  cksum = db->value( 8 ).toString();
592  recsize = db->value( 9 ).toString();
593  rawGUID = db->value( 0 ).toString();
594  comment = db->value( 3 ).toString();
595  }
596 
597  else
598  {
599  experID = db->value( 3 ).toString();
600  date = US_Util::toUTCDatetimeText( db->value( 5 )
601  .toDateTime().toString( Qt::ISODate ), true );
602  cksum = db->value( 6 ).toString();
603  recsize = db->value( 7 ).toString();
604  rawGUID = db->value( 9 ).toString();
605  comment = db->value( 10 ).toString();
606  expGUID = db->value( 11 ).toString();
607  }
608 
609  rawGUIDs[ recID ] = rawGUID;
610  irecID = recID.toInt();
611 DbgLv(1) << "BrDb: RAW id" << recID << " expID" << experID;
612  QString subType = "";
613  contents = cksum + " " + recsize;
614 
615  if ( comment.isEmpty() )
616  comment = filename.section( ".", 0, -2 );
617 
618  if ( ! label.contains( "." ) )
619  label = filename.section( ".", 0, -2 );
620 
621  if ( ! rfilt )
622  rawIDs << recID;
623 
624 DbgLv(2) << "BrDb: raw expGid" << expGUID;
625 DbgLv(2) << "BrDb: label filename comment" << label << filename << comment;
626 
627 //DbgLv(2) << "BrDb: (R)contents" << contents;
628 
629  cdesc.recordID = irecID;
630  cdesc.recType = 1;
631  cdesc.subType = subType;
632  cdesc.recState = REC_DB;
633  cdesc.dataGUID = rawGUID.simplified();
634  cdesc.parentGUID = expGUID.simplified();
635  cdesc.parentID = experID.toInt();
636  cdesc.filename = filename;
637  cdesc.contents = contents;
638  cdesc.label = label;
639  cdesc.description = comment;
640  cdesc.filemodDate = "";
641  cdesc.lastmodDate = date;
642 
643  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
644  cdesc.dataGUID = US_Util::new_guid();
645 
646  cdesc.parentGUID = cdesc.parentGUID.length() == 36 ?
647  cdesc.parentGUID : dmyGUID;
648 
649  ddescs << cdesc;
650  istep += incre;
651  progress->setValue( istep );
652  qApp->processEvents();
653  }
654  }
655 
656  int kraw = rawIDs.size();
657  if ( rfilt && kraw == 1 )
658  {
659  nstep /= qMax( nraws, 1 );
660  progress->setMaximum( nstep );
661  }
662  // get edited data IDs
663  lb_status->setText( tr( "Reading Edits" ) );
664  qApp->processEvents();
665  nqry = rfilt ? edtIDs.size() : qMin( nedts, 1 );
666  rfilt_q = rfilt && ( nqry < max_qrec );
667  nqry = rfilt_q ? nqry : qMin( nedts, 1 );
668 
669 DbgLv(1) << "BrDb: Query Edits" << nowTime() << "nqry" << nqry;
670  for ( int jq = 0; jq < nqry; jq++ )
671  {
672  QString edtID;
673  query.clear();
674  if ( rfilt_q )
675  {
676  edtID = edtIDs[ jq ];
677  query << "get_editedData" << edtID;
678  }
679  else
680  {
681  query << "all_editedDataIDs" << invID;
682  }
683  db->query( query );
684 
685  while ( db->next() )
686  { // Read Edit records
687  QString label;
688  QString filename;
689  QString filebase;
690  QString runID;
691  QString triple;
692  QString experID;
693  QString date;
694  QString cksum;
695  QString recsize;
696  QString comment;
697  QString rawID;
698  QString editGUID;
699 
700  if ( rfilt_q )
701  {
702  recID = edtID;
703  rawID = db->value( 0 ).toString();
704  editGUID = db->value( 1 ).toString();
705  label = db->value( 2 ).toString();
706  filename = db->value( 3 ).toString().replace( "\\", "/" );
707  filebase = filename.section( "/", -1, -1 );
708  date = US_Util::toUTCDatetimeText( db->value( 5 )
709  .toDateTime().toString( Qt::ISODate ), true );
710  comment = db->value( 4 ).toString();
711  cksum = db->value( 6 ).toString();
712  recsize = db->value( 7 ).toString();
713  runID = filebase.section( ".", 0, 0 );
714  triple = filebase.section( ".", -4, -2 );
715  if ( runID != filt_run ) continue;
716  if ( tfilt && triple != filt_triple ) continue;
717  }
718  else
719  {
720  recID = db->value( 0 ).toString();
721  label = db->value( 1 ).toString();
722  filename = db->value( 2 ).toString().replace( "\\", "/" );
723  filebase = filename.section( "/", -1, -1 );
724  rawID = db->value( 3 ).toString();
725  expID = db->value( 4 ).toString();
726  date = US_Util::toUTCDatetimeText( db->value( 5 )
727  .toDateTime().toString( Qt::ISODate ), true );
728  cksum = db->value( 6 ).toString();
729  recsize = db->value( 7 ).toString();
730  editGUID = db->value( 9 ).toString();
731  comment = "";
732  runID = filebase.section( ".", 0, 0 );
733  triple = filebase.section( ".", -4, -2 );
734  if ( rfilt && runID != filt_run ) continue;
735  if ( tfilt && triple != filt_triple ) continue;
736  }
737 
738  irecID = recID.toInt();
739 DbgLv(2) << "BrDb: EDT id" << recID << " raID" << db->value(3).toString()
740  << " expID" << db->value(4).toString();
741  rawGUID = rawGUIDs[ rawID ];
742 
743  if ( ! rfilt )
744  edtIDs << recID;
745 
746  QString subType = filebase.section( ".", 2, 2 );
747  contents = cksum + " " + recsize;
748 DbgLv(2) << "BrDb: edt id eGID rGID label date"
749  << irecID << editGUID << rawGUID << label << date;
750 //DbgLv(2) << "BrDb: (E)contents" << contents;
751 
752  if ( ! filename.contains( "/" ) )
753  filename = US_Settings::resultDir() + "/"
754  + filename.section( ".", 0, 0 ) + "/"
755  + filename;
756 //DbgLv(2) << "BrDb: fname" << filename;
757 
758  cdesc.recordID = irecID;
759  cdesc.recType = 2;
760  cdesc.subType = subType;
761  cdesc.recState = REC_DB;
762  cdesc.dataGUID = editGUID.simplified();
763  cdesc.parentGUID = rawGUID.simplified();
764  cdesc.parentID = rawID.toInt();
765  cdesc.filename = filename;
766  cdesc.contents = contents;
767  cdesc.description = ( comment.isEmpty() ) ?
768  filebase.section( ".", 0, 2 ) :
769  comment;
770  cdesc.label = cdesc.description;
771  cdesc.filemodDate = "";
772  cdesc.lastmodDate = date;
773 
774  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
775  cdesc.dataGUID = US_Util::new_guid();
776 
777  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
778  cdesc.parentGUID.simplified() : dmyGUID;
779  edtMap[ cdesc.dataGUID ] = cdesc.recordID; // save edit ID for GUID
780 
781  ddescs << cdesc;
782  istep += incre;
783  progress->setValue( istep );
784  qApp->processEvents();
785  }
786  }
787 DbgLv(1) << "BrDb: EDT loop done";
788 
789  // get model IDs
790  const int _M_LARGE_ = 65000; // Model large size indicating CUSTOMGRID
791  QStringList tmodels;
792  QList< int > tmodnxs;
793  lb_status->setText ( tr( "Reading Models" ) );
794  progress ->setValue( istep );
795  qApp->processEvents();
796 DbgLv(1) << "BrDb: Reading Models";
797  nqry = rfilt ? edtIDs.size() : 1;
798  rfilt_q = rfilt && ( nqry < max_qrec );
799  nqry = rfilt_q ? nqry : qMin( nedts, 1 );
800 
801 DbgLv(1) << "BrDb: Query Models" << nowTime();
802  for ( int jq = 0; jq < nqry; jq++ )
803  {
804  query.clear();
805 
806  if ( rfilt_q )
807  query << "get_model_desc_by_editID" << invID << edtIDs[ jq ];
808  else
809  query << "get_model_desc" << invID;
810 
811 DbgLv(2) << "BrDb: Query Models" << nowTime();
812 int kmdl=0;
813  db->query( query );
814 DbgLv(2) << "BrDb: Query Return" << nowTime();
815 
816  while ( db->next() )
817  { // get model information from DB
818  recID = db->value( 0 ).toString();
819  QString editID = db->value( 6 ).toString();
820 if( (++kmdl) == 1 )
821 DbgLv(2) << "BrDb: First Model " << nowTime();
822  if ( rfilt )
823  {
824  if ( ! edtIDs.contains( editID ) ) continue;
825  }
826  else
827  modIDs << recID;
828 
829  irecID = recID.toInt();
830  QString modelGUID = db->value( 1 ).toString();
831  QString descript = db->value( 2 ).toString();
832  modDescs << descript;
833 
834  if ( descript.length() == 80 )
835  { // Truncated description? save for later testing/replacement
836  tmodels << recID;
837  tmodnxs << ddescs.size();
838  }
839 
840  QString editGUID = db->value( 5 ).toString();
841 DbgLv(2) << "BrDb: MOD id" << recID << " edID" << editID << " edGID" << editGUID;
842 DbgLv(2) << "BrDb: MOD id" << recID << " desc" << descript;
843  QString date = US_Util::toUTCDatetimeText( db->value( 7 )
844  .toDateTime().toString( Qt::ISODate ), true );
845  QString cksum = db->value( 8 ).toString();
846  QString recsize = db->value( 9 ).toString();
847  QString label = descript.section( ".", 0, -2 );
848 
849  if ( label.length() > 40 )
850  label = label.left( 13 ) + "..." + label.right( 24 );
851 
852  // Get the sub-analysis-type
853  QString subType = descript.section( ".", -2, -2 ).section( "_",2,2 );
854 
855  // Set as CUSTOMGRID if so marked or large non-MC
856  if ( descript.contains( "CustomGrid" ) ||
857  ( !descript.contains( "_mc" ) && recsize.toInt() > _M_LARGE_ ) )
858  subType = "CUSTOMGRID";
859 
860  // If empty subtype, mark as MANUAL
861  else if ( subType.isEmpty() )
862  subType = "MANUAL";
863 
864  contents = cksum + " " + recsize;
865 //DbgLv(2) << "BrDb: det: cont" << contents;
866  cdesc.recordID = irecID;
867  cdesc.recType = 3;
868  cdesc.subType = subType;
869  cdesc.recState = REC_DB;
870  cdesc.dataGUID = modelGUID.simplified();
871  cdesc.parentGUID = editGUID;
872  cdesc.parentID = edtMap[ editGUID ];
873  cdesc.filename = "";
874  cdesc.contents = contents;
875  cdesc.label = label;
876  cdesc.description = descript;
877  cdesc.filemodDate = "";
878  cdesc.lastmodDate = date;
879 
880  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
881  cdesc.dataGUID = US_Util::new_guid();
882 
883  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
884  cdesc.parentGUID.simplified() : dmyGUID;
885 
886  ddescs << cdesc;
887  progress->setValue( ++istep );
888  qApp->processEvents();
889  }
890  }
891 DbgLv(2) << "BrDb: Last Model " << nowTime();
892 
893  // Get noise IDs
894  QStringList tnoises;
895  QList< int > tnoinxs;
896  lb_status->setText( tr( "Reading Noises" ) );
897  qApp->processEvents();
898 
899  for ( int jq = 0; jq < nqry; jq++ )
900  {
901  query.clear();
902 
903  if ( rfilt_q )
904  query << "get_noise_desc_by_editID" << invID << edtIDs[ jq ];
905  else
906  query << "get_noise_desc" << invID;
907 
908 DbgLv(2) << "BrDb: Query Noises" << nowTime();
909  db->query( query );
910 
911  while ( db->next() )
912  { // Get noise information from DB
913  recID = db->value( 0 ).toString();
914  irecID = recID.toInt();
915  QString editID = db->value( 2 ).toString();
916  if ( rfilt )
917  {
918  if ( ! edtIDs.contains( editID ) ) continue;
919  }
920  else
921  noiIDs << recID;
922 
923  QString noiseGUID = db->value( 1 ).toString();
924  QString modelID = db->value( 3 ).toString();
925  QString noiseType = db->value( 4 ).toString();
926  QString modelGUID = db->value( 5 ).toString();
927  QString date = US_Util::toUTCDatetimeText( db->value( 6 )
928  .toDateTime().toString( Qt::ISODate ), true );
929  QString cksum = db->value( 7 ).toString();
930  QString recsize = db->value( 8 ).toString();
931  QString descript = db->value( 9 ).toString();
932 DbgLv(2) << "BrDb: NOI id" << recID << " edID" << editID << " moID" << modelID
933  << " descript" << descript;
934 
935  if ( descript.isEmpty() || descript.length() == 80 )
936  {
937  int jmod = modIDs.indexOf( modelID );
938  if ( jmod >= 0 )
939  {
940  descript = modDescs.at( jmod );
941 
942  if ( descript.length() == 80 )
943  { // Truncated description? Save for later review/replace
944  tnoises << recID;
945  tnoinxs << ddescs.size();
946  }
947 
948  descript = descript.replace( ".model", "." + noiseType );
949 DbgLv(2) << "BrDb: jmod" << jmod << " descript" << descript;
950  }
951  }
952 //DbgLv(3) << "BrDb: contents================================================";
953 //DbgLv(3) << contents.left( 200 );
954 //DbgLv(3) << "BrDb: contents================================================";
955 
956  contents = cksum + " " + recsize;
957  QString label = descript.section( ".", 0, -2 );
958 
959  if ( label.length() > 40 )
960  label = label.left( 13 ) + "..." + label.right( 24 );
961 
962  cdesc.recordID = irecID;
963  cdesc.recType = 4;
964  cdesc.subType = ( noiseType == "ti_noise" ) ? "TI" : "RI";
965  cdesc.recState = REC_DB;
966  cdesc.dataGUID = noiseGUID.simplified();
967  cdesc.parentGUID = modelGUID.simplified();
968  cdesc.parentID = modelID.toInt();
969  cdesc.filename = "";
970  cdesc.contents = contents;
971  cdesc.label = label;
972  cdesc.description = descript;
973  cdesc.filemodDate = "";
974  cdesc.lastmodDate = date;
975 DbgLv(2) << "BrDb: noi id nGID dsc typ noityp"
976  << irecID << noiseGUID << descript << cdesc.subType << noiseType;
977 
978  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
979  cdesc.dataGUID = US_Util::new_guid();
980 
981  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
982  cdesc.parentGUID.simplified() : dmyGUID;
983 
984  ddescs << cdesc;
985  progress->setValue( ++istep );
986  qApp->processEvents();
987  }
988 DbgLv(2) << "BrDb: Noise IDs" << nowTime() << "size" << noiIDs.size();
989  }
990 
991  for ( int ii = 0; ii < tmodels.size(); ii++ )
992  { // Change truncated model descriptions
993  recID = tmodels[ ii ];
994  int jdsc = tmodnxs[ ii ];
995  cdesc = ddescs.at( jdsc );
996  US_Model model1;
997  model1.load( recID, db );
998  QString descript = model1.description;
999  QString label = descript.section( ".", 0, -2 );
1000 
1001  if ( label.length() > 40 )
1002  label = label.left( 13 ) + "..." + label.right( 24 );
1003 
1004 DbgLv(2) << "BrDb: ii jdsc" << ii << jdsc << "dsc1" << cdesc.description
1005  << "dsc2" << descript;
1006  cdesc.description = descript;
1007  cdesc.label = label;
1008  ddescs.replace( jdsc, cdesc );
1009  }
1010 
1011  for ( int ii = 0; ii < tnoises.size(); ii++ )
1012  { // Change truncated noise descriptions
1013  recID = tnoises[ ii ];
1014  int jdsc = tnoinxs[ ii ];
1015  cdesc = ddescs.at( jdsc );
1016  US_Noise noise1;
1017  noise1.load( recID, db );
1018  QString descript = noise1.description;
1019  QString label = descript.section( ".", 0, -2 );
1020 
1021  if ( label.length() > 40 )
1022  label = label.left( 13 ) + "..." + label.right( 24 );
1023 
1024 DbgLv(2) << "BrDb: ii jdsc" << ii << jdsc << "dsc1" << cdesc.description
1025  << "dsc2" << descript;
1026  cdesc.description = descript;
1027  cdesc.label = label;
1028  ddescs.replace( jdsc, cdesc );
1029  }
1030 
1031 
1032  progress->setMaximum( nstep );
1033  qApp->processEvents();
1034 DbgLv(1) << "BrDb: kr ke km kn"
1035  << rawIDs.size() << edtIDs.size() << modIDs.size() << noiIDs.size();
1036 DbgLv(1) << "BrDb: scan time:"
1037  << basetime.msecsTo(QDateTime::currentDateTime())/1000.0;
1038 
1039  progress->setValue( nstep );
1040  lb_status->setText( tr( "Database Review Complete" ) );
1041  qApp->processEvents();
1042 }
1043 
1044 // scan the local disk for R/E/M/N data sets
1045 void US_DataModel::scan_local( )
1046 {
1047  ldescs.clear(); // local descriptions
1048  adescs.clear(); // all descriptions
1049 
1050  if ( ! filt_source.isEmpty() && filt_source == "DB Only" )
1051  { // If source filter is "DB-only", skip Local scan
1052  return;
1053  }
1054 
1055  // start with AUC (raw) and edit files in directories of resultDir
1056  bool rfilt = ( ! filt_run .isEmpty() && filt_run != "ALL" );
1057  bool tfilt = ( ! filt_triple.isEmpty() && filt_triple != "ALL" );
1058  QString rdir = US_Settings::resultDir();
1059 // rdir = rfilt ? ( rdir + "/" + filt_run ) : rdir;
1060  QString ddir = US_Settings::dataDir();
1061  QString dirm = ddir + "/models";
1062  QString dirn = ddir + "/noises";
1063  QString contents = "";
1064  QString dmyGUID = "00000000-0000-0000-0000-000000000000";
1065  QStringList aucdirs = rfilt ? QStringList( filt_run )
1066  : QDir( rdir )
1067  .entryList( QDir::AllDirs | QDir::NoDotAndDotDot, QDir::Name );
1068 
1069  QStringList aucfilt;
1070  QStringList edtfilt;
1071  QStringList edtIDs;
1072  QStringList mdlIDs;
1073  QStringList modfilt( "M*xml" );
1074  QStringList noifilt( "N*xml" );
1075  QStringList modfils = QDir( dirm )
1076  .entryList( modfilt, QDir::Files, QDir::Name );
1077  QStringList noifils = QDir( dirn )
1078  .entryList( noifilt, QDir::Files, QDir::Name );
1079  int ktask = 0;
1080  int naucd = aucdirs.size();
1081  int nedtf = naucd * 3;
1082  int nmodf = modfils.size();
1083  int nnoif = noifils.size();
1084  QString aucpatt = "*.auc";
1085  QString edtpatt = "*.xml";
1086 
1087  if ( rfilt )
1088  { // If run filtering, count actual files that match
1089  nedtf = 0;
1090  nmodf = 0;
1091  nnoif = 0;
1092 
1093  for ( int ii = 0; ii < naucd; ii++ )
1094  {
1095  QString subdir = rdir + aucdirs.at( ii );
1096  edtpatt = tfilt ?
1097  filt_run + ".*" + filt_triple + ".xml" :
1098  filt_run + ".*.xml";
1099  edtfilt.clear();
1100  edtfilt << edtpatt;
1101  QStringList edtfiles = QDir( subdir )
1102  .entryList( edtfilt, QDir::Files, QDir::Name );
1103  nedtf += edtfiles.size();
1104  aucpatt = tfilt ?
1105  filt_run + ".*" + filt_triple + ".auc" :
1106  filt_run + ".*.auc";
1107  }
1108 
1109  for ( int ii = 0; ii < modfils.size(); ii++ )
1110  {
1111  US_Model model;
1112  QString modfil = dirm + "/" + modfils.at( ii );
1113  model.load( modfil );
1114  QString mdesc = model.description;
1115  if ( ! mdesc.startsWith( filt_run ) ) continue;
1116  if ( tfilt && ! mdesc.contains( filt_triple ) ) continue;
1117  nmodf++;
1118  }
1119 
1120  for ( int ii = 0; ii < noifils.size(); ii++ )
1121  {
1122  US_Noise noise;
1123  QString noifil = dirn + "/" + noifils.at( ii );
1124  noise.load( noifil );
1125  QString ndesc = noise.description;
1126  if ( ! ndesc.startsWith( filt_run ) ) continue;
1127  if ( tfilt && ! ndesc.contains( filt_triple ) ) continue;
1128  nnoif++;
1129  }
1130  }
1131 
1132  int nstep = naucd + nedtf + nmodf + nnoif;
1133 DbgLv(1) << "BrLoc: naucd nedtf nmodf nnoif nstep"
1134  << naucd << nedtf << nmodf << nnoif << nstep;
1135  aucfilt.clear();
1136  edtfilt.clear();
1137  aucfilt << aucpatt;
1138  edtfilt << edtpatt;
1139 DbgLv(1) << "BrLoc: aucfilt" << aucfilt << "edtfilt" << edtfilt;
1140  rdir = rdir + "/";
1141  lb_status->setText( tr( "Reading Local-Disk Data..." ) );
1142  progress->setMaximum( nstep );
1143  qApp->processEvents();
1144 
1145  for ( int ii = 0; ii < naucd; ii++ )
1146  { // loop thru potential data directories
1147  QString subdir = rdir + aucdirs.at( ii );
1148  QStringList aucfiles = QDir( subdir )
1149  .entryList( aucfilt, QDir::Files, QDir::Name );
1150  int naucf = aucfiles.size();
1151 DbgLv(1) << "BrLoc: ii naucf" << ii << naucf << "subdir" << subdir;
1152  US_DataIO::RawData rdata;
1153  US_DataIO::EditValues edval;
1154 
1155  for ( int jj = 0; jj < naucf; jj++ )
1156  { // loop thru .auc files found in a directory
1157  QString fname = aucfiles.at( jj );
1158  QString runid = fname.section( ".", 0, 0 );
1159  QString tripl = fname.section( ".", -5, -2 );
1160  QString aucfile = subdir + "/" + fname;
1161  QString descr = "";
1162  QString expGUID = expGUIDauc( aucfile );
1163 DbgLv(2) << "BrLoc: ii jj file" << ii << jj << aucfile;
1164 
1165  // read in the raw data and build description record
1166  US_DataIO::readRawData( aucfile, rdata );
1167 
1168  contents = US_Util::md5sum_file( aucfile );
1169 DbgLv(2) << "BrLoc: contents" << contents;
1170 
1171  QString uuid = US_Util::uuid_unparse( (uchar*)rdata.rawGUID );
1172  QString rawGUID = uuid;
1173 
1174  cdesc.recordID = -1;
1175  cdesc.recType = 1;
1176  cdesc.subType = "";
1177  cdesc.recState = REC_LO;
1178  cdesc.dataGUID = rawGUID.simplified();
1179  cdesc.parentGUID = expGUID.simplified();
1180  cdesc.parentID = -1;
1181  cdesc.filename = aucfile;
1182  cdesc.contents = contents;
1183  cdesc.label = runid + "." + tripl;
1184  cdesc.description = rdata.description;
1185  cdesc.filemodDate = US_Util::toUTCDatetimeText( QFileInfo( aucfile )
1186  .lastModified().toUTC().toString( Qt::ISODate )
1187  , true );
1188  cdesc.lastmodDate = "";
1189 
1190  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
1191  cdesc.dataGUID = US_Util::new_guid();
1192 
1193  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
1194  cdesc.parentGUID.simplified() : dmyGUID;
1195 
1196  ldescs << cdesc;
1197 
1198  // now load edit files associated with this auc file
1199  edtfilt.clear();
1200  edtfilt << runid + ".*." + tripl + ".xml";
1201 DbgLv(2) << "BrLoc: edtfilt" << edtfilt;
1202 
1203  QStringList edtfiles = QDir( subdir )
1204  .entryList( edtfilt, QDir::Files, QDir::Name );
1205 
1206  for ( int kk = 0; kk < edtfiles.size(); kk++ )
1207  {
1208  QString efname = edtfiles.at( kk );
1209  QString editid = efname.section( ".", 1, 3 );
1210  QString edtfile = subdir + "/" + efname;
1211  contents = "";
1212 //DbgLv(2) << "BrLoc: kk file" << kk << edtfile;
1213 
1214  // read EditValues for the edit data and build description record
1215  US_DataIO::readEdits( edtfile, edval );
1216 
1217  contents = US_Util::md5sum_file( edtfile );
1218 //DbgLv(2) << "BrLoc: (E)contents edtfile" << contents << edtfile;
1219 
1220  cdesc.recordID = -1;
1221  cdesc.recType = 2;
1222  cdesc.subType = efname.section( ".", 2, 2 );
1223  cdesc.recState = REC_LO;
1224  cdesc.dataGUID = edval.editGUID.simplified();
1225  cdesc.parentGUID = edval.dataGUID.simplified();
1226  cdesc.parentID = -1;
1227  cdesc.filename = edtfile;
1228  cdesc.contents = contents;
1229  cdesc.label = runid + "." + editid;
1230  cdesc.description = efname.section( ".", 0, -2 );
1231  cdesc.filemodDate = US_Util::toUTCDatetimeText( QFileInfo( edtfile )
1232  .lastModified().toUTC().toString( Qt::ISODate )
1233  , true );
1234  cdesc.lastmodDate = "";
1235 
1236  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
1237  cdesc.dataGUID = US_Util::new_guid();
1238 
1239  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
1240  cdesc.parentGUID.simplified() : dmyGUID;
1241 
1242  ldescs << cdesc;
1243  edtIDs << cdesc.dataGUID;
1244  }
1245  if ( ii == ( naucd / 2 ) && jj == ( naucf / 2 ) )
1246  {
1247  progress->setValue( ++ktask );
1248  qApp->processEvents();
1249  }
1250  }
1251  progress->setValue( ++ktask );
1252  qApp->processEvents();
1253  }
1254  progress->setValue( ++ktask );
1255  qApp->processEvents();
1256 
1257  for ( int ii = 0; ii < nmodf; ii++ )
1258  { // loop thru potential model files
1259  US_Model model;
1260  QString modfil = dirm + "/" + modfils.at( ii );
1261  contents = "";
1262 
1263  model.load( modfil );
1264 
1265  contents = US_Util::md5sum_file( modfil );
1266 
1267  cdesc.recordID = -1;
1268  cdesc.recType = 3;
1269  cdesc.subType = model_type( model );
1270  cdesc.recState = REC_LO;
1271  cdesc.dataGUID = model.modelGUID.simplified();
1272  cdesc.parentGUID = model.editGUID.simplified();
1273 
1274  if ( rfilt && ! edtIDs.contains( cdesc.parentGUID ) ) continue;
1275 
1276  cdesc.parentID = -1;
1277  cdesc.filename = modfil;
1278  cdesc.contents = contents;
1279  cdesc.description = model.description;
1280  cdesc.filemodDate = US_Util::toUTCDatetimeText( QFileInfo( modfil )
1281  .lastModified().toUTC().toString( Qt::ISODate )
1282  , true );
1283  cdesc.lastmodDate = "";
1284  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
1285  cdesc.dataGUID = US_Util::new_guid();
1286 
1287  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
1288  cdesc.parentGUID.simplified() : dmyGUID;
1289  QString label = model.description.section( ".", 0, -2 );
1290  cdesc.label = ( label.length() < 41 ) ? label :
1291  ( label.left( 13 ) + "..." + label.right( 24 ) );
1292 
1293  ldescs << cdesc;
1294  mdlIDs << cdesc.dataGUID;
1295 
1296  progress->setValue( ++ktask );
1297  qApp->processEvents();
1298  }
1299 
1300  for ( int ii = 0; ii < nnoif; ii++ )
1301  { // loop thru potential noise files
1302  US_Noise noise;
1303  QString noifil = dirn + "/" + noifils.at( ii );
1304 
1305  noise.load( noifil );
1306 
1307  contents = US_Util::md5sum_file( noifil );
1308 
1309  cdesc.recordID = -1;
1310  cdesc.recType = 4;
1311  cdesc.subType = ( noise.type == US_Noise::RI ) ? "RI" : "TI";
1312  cdesc.recState = REC_LO;
1313  cdesc.dataGUID = noise.noiseGUID.simplified();
1314  cdesc.parentGUID = noise.modelGUID.simplified();
1315 
1316  if ( rfilt && ! mdlIDs.contains( cdesc.parentGUID ) ) continue;
1317 
1318  cdesc.parentID = -1;
1319  cdesc.filename = noifil;
1320  cdesc.contents = contents;
1321  cdesc.description = noise.description;
1322  cdesc.filemodDate = US_Util::toUTCDatetimeText( QFileInfo( noifil )
1323  .lastModified().toUTC().toString( Qt::ISODate )
1324  , true );
1325  cdesc.lastmodDate = "";
1326 
1327  if ( cdesc.dataGUID.length() != 36 || cdesc.dataGUID == dmyGUID )
1328  cdesc.dataGUID = US_Util::new_guid();
1329 
1330  cdesc.parentGUID = cdesc.parentGUID.simplified().length() == 36 ?
1331  cdesc.parentGUID.simplified() : dmyGUID;
1332  QString label = noise.description;
1333  cdesc.label = ( label.length() < 41 ) ? label :
1334  ( label.left( 9 ) + "..." + label.right( 28 ) );
1335 
1336  ldescs << cdesc;
1337 
1338  progress->setValue( ++ktask );
1339  qApp->processEvents();
1340  }
1341 
1342  progress->setValue( nstep );
1343  lb_status->setText( tr( "Local Data Review Complete" ) );
1344  qApp->processEvents();
1345 }
1346 
1347 // merge the database and local description vectors into a single combined
1348 void US_DataModel::merge_dblocal( )
1349 {
1350  int nddes = ddescs.size();
1351  int nldes = ldescs.size();
1352  int nstep = ( ( nddes + nldes ) * 5 ) / 8;
1353 
1354  int jdr = 0;
1355  int jlr = 0;
1356  int kar = 1;
1357 
1358  DataDesc descd = ( nddes > 0 ) ? ddescs.at( 0 ) : DataDesc();
1359  DataDesc descl = ( nldes > 0 ) ? ldescs.at( 0 ) : DataDesc();
1360 DbgLv(1) << "MERGE: nd nl dlab llab"
1361  << nddes << nldes << descd.label << descl.label;
1362 
1363  lb_status->setText( tr( "Merging Data ..." ) );
1364  progress->setMaximum( nstep );
1365  qApp->processEvents();
1366 
1367  while ( jdr < nddes && jlr < nldes )
1368  { // main loop to merge records until one is exhausted
1369 
1370  progress->setValue( kar ); // report progress
1371 
1372  if ( kar > nstep )
1373  { // if count beyond max, bump max by one eighth
1374  nstep = ( kar * 9 ) / 8;
1375  progress->setMaximum( nstep );
1376  }
1377  qApp->processEvents();
1378 
1379  while ( descd.dataGUID == descl.dataGUID )
1380  { // records match in GUID: merge them into one
1381  descd.recState |= descl.recState; // OR states
1382  descd.filename = descl.filename; // filename from local
1383  descd.filemodDate = descl.filemodDate; // file last mod from local
1384  descd.description = descl.description; // description from local
1385 //if ( descl.recType == 3 && descd.contents != descl.contents ) {
1386 // US_Model modell;
1387 // US_Model modeld;
1388 // modell.load( descl.filename );
1389 // modeld.load( QString::number(descd.recordID), db );
1390 // DbgLv(1) << " ++LOCAL Model:";
1391 // modell.debug();
1392 // DbgLv(1) << " ++DB Model:";
1393 // modeld.debug(); }
1394  descd.contents = descd.contents + " " + descl.contents;
1395 
1396  adescs << descd; // output combo record
1397 DbgLv(2) << "MERGE: kar jdr jlr (1)GID" << kar << jdr << jlr << descd.dataGUID;
1398  kar++;
1399 
1400  if ( ++jdr < nddes ) // bump db count and test if done
1401  descd = ddescs.at( jdr ); // get next db record
1402 
1403  else
1404  {
1405  if ( ++jlr < nldes )
1406  descl = ldescs.at( jlr ); // get next local record
1407  break;
1408  }
1409 
1410 
1411  if ( ++jlr < nldes ) // bump local count and test if done
1412  descl = ldescs.at( jlr ); // get next local record
1413  else
1414  break;
1415  }
1416 
1417  if ( jdr >= nddes || jlr >= nldes )
1418  break;
1419 
1420  while ( descd.recType > descl.recType )
1421  { // output db records that are left-over children
1422  adescs << descd;
1423 DbgLv(2) << "MERGE: kar jdr jlr (2)GID" << kar << jdr << jlr << descd.dataGUID;
1424  kar++;
1425 
1426  if ( ++jdr < nddes )
1427  descd = ddescs.at( jdr );
1428  else
1429  break;
1430  }
1431 
1432  if ( jdr >= nddes || jlr >= nldes )
1433  break;
1434 
1435  while ( descl.recType > descd.recType )
1436  { // output local records that are left-over children
1437  adescs << descl;
1438 DbgLv(2) << "MERGE: kar jdr jlr (3)GID" << kar << jdr << jlr << descl.dataGUID;
1439  kar++;
1440 
1441  if ( ++jlr < nldes )
1442  descl = ldescs.at( jlr );
1443  else
1444  break;
1445  }
1446 
1447  if ( jdr >= nddes || jlr >= nldes )
1448  break;
1449 
1450  // If we've reached another matching pair or if we are not at
1451  // the same level, go back up to the start of the main loop.
1452  if ( descd.dataGUID == descl.dataGUID ||
1453  descd.recType != descl.recType )
1454  continue;
1455 
1456  // If we are here, we have records at the same level,
1457  // but with different GUIDs. Output one of them, based on
1458  // an alphanumeric comparison of label values.
1459 
1460  QString dlabel = descd.label;
1461  QString llabel = descl.label;
1462  if ( descd.recType > 2 )
1463  {
1464  dlabel = descd.description;
1465  llabel = descl.description;
1466  }
1467 DbgLv(2) << "MERGE: rtype dlabel llabel" << descd.recType << dlabel << llabel;
1468 
1469  if ( dlabel < llabel )
1470  { // output db record first based on alphabetic label sort
1471  adescs << descd;
1472 DbgLv(2) << "MERGE: kar jdr jlr (4)GID" << kar << jdr << jlr << descd.dataGUID;
1473  kar++;
1474 
1475  if ( ++jdr < nddes )
1476  descd = ddescs.at( jdr );
1477  else
1478  break;
1479  }
1480 
1481  else
1482  { // output local record first based on alphabetic label sort
1483  adescs << descl;
1484 DbgLv(2) << "MERGE: kar jdr jlr (5)GID" << kar << jdr << jlr << descl.dataGUID;
1485  kar++;
1486 
1487  if ( ++jlr < nldes )
1488  descl = ldescs.at( jlr );
1489  else
1490  break;
1491  }
1492 
1493  } // end of main merge loop;
1494 
1495  // after breaking from main loop, output any records left from one
1496  // source (db/local) or the other.
1497  nstep += ( nddes - jdr + nldes - jlr );
1498  progress->setMaximum( nstep );
1499  qApp->processEvents();
1500 
1501  while ( jdr < nddes )
1502  {
1503  adescs << ddescs.at( jdr++ );
1504 descd=ddescs.at(jdr-1);
1505 DbgLv(2) << "MERGE: kar jdr jlr (8)GID" << kar << jdr << jlr << descd.dataGUID;
1506  progress->setValue( ++kar );
1507  qApp->processEvents();
1508  }
1509 
1510  while ( jlr < nldes )
1511  {
1512  adescs << ldescs.at( jlr++ );
1513 descl=ldescs.at(jlr-1);
1514 DbgLv(2) << "MERGE: kar jdr jlr (9)GID" << kar << jdr << jlr << descl.dataGUID;
1515  progress->setValue( ++kar );
1516  qApp->processEvents();
1517  }
1518 
1519 DbgLv(2) << "MERGE: nddes nldes kar" << nddes << nldes << --kar;
1520 DbgLv(2) << " a/d/l sizes" << adescs.size() << ddescs.size() << ldescs.size();
1521 
1522  progress->setValue( nstep );
1523  lb_status->setText( tr( "Data Merge Complete" ) );
1524  qApp->processEvents();
1525 }
1526 
1527 // Exclude DB-only or Local-only trees
1528 void US_DataModel::exclude_trees( )
1529 {
1530  QVector< DataDesc > tdess = adescs; // temporary descr. vector
1531  DataDesc desct; // temporary descr. entry
1532  int krecs = tdess.size();
1533  int nrecs = 0;
1534  bool excdb = filt_source.contains( "DB" );
1535  bool exctr = false;
1536  adescs.clear();
1537 
1538  for ( int ii = 0; ii < krecs; ii++ )
1539  {
1540  desct = tdess[ ii ];
1541 
1542  if ( desct.recType == 1 )
1543  { // If head of tree, test whether DB/Local/Both
1544  bool isDba = ( ( desct.recState & REC_DB ) != 0 );
1545  bool isLoc = ( ( desct.recState & REC_LO ) != 0 );
1546 
1547  if ( isDba && isLoc )
1548  { // If both, tree should not be excluded
1549  exctr = FALSE;
1550  }
1551 
1552  else if ( isDba )
1553  { // If DB-Only, exclude by Exclude-DB-Only filter
1554  exctr = excdb;
1555  }
1556 
1557  else
1558  { // If Local-Only, exclude by Exclude-Local-Only filter
1559  exctr = ! excdb;
1560  }
1561  }
1562 
1563  if ( exctr )
1564  { // Tree is excluded, so skip this record
1565  continue;
1566  }
1567 
1568  // Otherwise, copy to re-created descriptions vector
1569  adescs << desct;
1570  nrecs++;
1571  }
1572 }
1573 
1574 // sort a data-set description vector
1575 void US_DataModel::sort_descs( QVector< DataDesc >& descs )
1576 {
1577  QVector< DataDesc > tdess; // temporary descr. vector
1578  DataDesc desct; // temporary descr. entry
1579  QStringList sortr; // sort string lists
1580  QStringList sorte;
1581  QStringList sortm;
1582  QStringList sortn;
1583  int nrecs = descs.size(); // number of descr. records
1584 
1585  lb_status->setText( tr( "Sorting Descriptions..." ) );
1586  qApp->processEvents();
1587 DbgLv(1) << "sort_desc: nrecs" << nrecs;
1588  if ( nrecs == 0 )
1589  return;
1590 
1591  tdess.resize( nrecs );
1592  // Determine maximum description string length
1593  maxdlen = 0;
1594  for ( int ii = 0; ii < nrecs; ii++ )
1595  maxdlen = qMax( maxdlen, descs[ ii ].description.length() );
1596 
1597  for ( int ii = 0; ii < nrecs; ii++ )
1598  { // build sort strings for Raw,Edit,Model,Noise; copy unsorted vector
1599  desct = descs[ ii ];
1600 
1601  if ( desct.recType == 1 )
1602  sortr << sort_string( desct, ii );
1603 
1604  else if ( desct.recType == 2 )
1605  sorte << sort_string( desct, ii );
1606 
1607  else if ( desct.recType == 3 )
1608  sortm << sort_string( desct, ii );
1609 
1610  else if ( desct.recType == 4 )
1611  sortn << sort_string( desct, ii );
1612 
1613  tdess[ ii ] = desct;
1614  }
1615 DbgLv(2) << "SrtD: nrecs" << nrecs << nowTime();
1616 
1617  // sort the string lists for each type
1618  sortr.sort();
1619  sorte.sort();
1620  sortm.sort();
1621  sortn.sort();
1622 DbgLv(2) << "SrtD: sort[remn]" << nowTime();
1623 
1624  lb_status->setText( tr( "Finding Duplicates..." ) );
1625  qApp->processEvents();
1626  // review each type for duplicate GUIDs
1627  if ( review_descs( sortr, tdess ) )
1628  return;
1629 DbgLv(2) << "SrtD: RD(r)" << nowTime();
1630  if ( review_descs( sorte, tdess ) )
1631  return;
1632 DbgLv(2) << "SrtD: RD(e)" << nowTime();
1633  if ( review_descs( sortm, tdess ) )
1634  return;
1635 DbgLv(2) << "SrtD: RD(m)" << nowTime();
1636  if ( review_descs( sortn, tdess ) )
1637  return;
1638 DbgLv(2) << "SrtD: RD(n)" << nowTime();
1639 
1640  lb_status->setText( tr( "Finding Orphans..." ) );
1641  qApp->processEvents();
1642 
1643  // create list of noise,model,edit orphans
1644  QStringList orphn = list_orphans( sortn, sortm );
1645  QStringList orphm = list_orphans( sortm, sorte );
1646  QStringList orphe = list_orphans( sorte, sortr );
1647 DbgLv(2) << "SrtD: Orph(nme)" << nowTime();
1648 
1649  QString dmyGUID = "00000000-0000-0000-0000-000000000000";
1650  QString dsorts;
1651  QString dlabel;
1652  QString dindex;
1653  QString ddGUID;
1654  QString dpGUID;
1655  QString ppGUID;
1656  int kndx = tdess.size();
1657  int jndx;
1658  int kk;
1659  int ndmy = 0; // flag of duplicate dummies
1660 
1661  // Create lists of parent GUIDs
1662  QStringList guidsr;
1663  QStringList guidse;
1664  QStringList guidsm;
1665 
1666  for ( int ii = 0; ii < sortr.size(); ii++ )
1667  guidsr << sortr.at( ii ).section( ":", 2, 2 ).simplified();
1668 
1669  for ( int ii = 0; ii < sorte.size(); ii++ )
1670  guidse << sorte.at( ii ).section( ":", 2, 2 ).simplified();
1671 
1672  for ( int ii = 0; ii < sortm.size(); ii++ )
1673  guidsm << sortm.at( ii ).section( ":", 2, 2 ).simplified();
1674 
1675  // create dummy records to parent each orphan
1676 
1677  int nstep = orphn.size() + orphm.size() + orphe.size();
1678  int istep = 0;
1679  progress->setMaximum( nstep );
1680  progress->setValue ( istep );
1681  qApp->processEvents();
1682 DbgLv(2) << "(1) orphan: N size M size" << orphn.size() << orphm.size();
1683  for ( int ii = 0; ii < orphn.size(); ii++ )
1684  { // for each orphan noise, create a dummy model
1685  dsorts = orphn.at( ii );
1686  dlabel = dsorts.section( ":", 0, 0 );
1687  dindex = dsorts.section( ":", 1, 1 );
1688  ddGUID = dsorts.section( ":", 2, 2 ).simplified();
1689  dpGUID = dsorts.section( ":", 3, 3 ).simplified();
1690  jndx = dindex.toInt();
1691  cdesc = tdess[ jndx ];
1692 
1693  if ( dpGUID.length() < 2 || dpGUID == dmyGUID )
1694  { // handle case where there is no valid parentGUID
1695  if ( ndmy == 0 ) // first time: create one
1696  dpGUID = dmyGUID;
1697  else
1698  dpGUID = ppGUID; // afterwards: re-use same parent
1699 
1700  kk = sortn.indexOf( dsorts ); // find index in full list
1701  dsorts = dlabel + ":" + dindex + ":" + ddGUID + ":" + dpGUID;
1702 
1703  if ( kk >= 0 )
1704  { // replace present record for new parentGUID
1705  sortn.replace( kk, dsorts );
1706  cdesc.parentGUID = dpGUID;
1707  tdess[ jndx ] = cdesc;
1708  }
1709 
1710  if ( ndmy > 0 ) // after 1st time, skip creating new parent
1711  continue;
1712 
1713  ndmy++; // flag that we have a parent for invalid ones
1714  ppGUID = dpGUID; // save the GUID for new dummy parent
1715  }
1716 
1717  // If this record is no longer an orphan, skip creating new parent
1718  if ( guidsm.indexOf( dpGUID ) >= 0 )
1719  continue;
1720 
1721  if ( dpGUID == dmyGUID )
1722  cdesc.label = "Dummy-Model-for-Orphans";
1723 
1724  cdesc.parentID = cdesc.recordID;
1725  cdesc.recordID = -1;
1726  cdesc.recType = 3;
1727  cdesc.subType = "";
1728  cdesc.recState = NOSTAT;
1729  cdesc.dataGUID = dpGUID;
1730  cdesc.parentGUID = dmyGUID;
1731  cdesc.parentID = -1;
1732  cdesc.filename = "";
1733  cdesc.contents = "";
1734  cdesc.label = cdesc.label.section( ".", 0, 0 );
1735  cdesc.description = cdesc.label + "--ARTIFICIAL-RECORD";
1736  cdesc.filemodDate = US_Util::toUTCDatetimeText(
1737  QDateTime::currentDateTime().toUTC()
1738  .toString( Qt::ISODate ), true );
1739  cdesc.lastmodDate = cdesc.filemodDate;
1740 
1741  dlabel = dlabel.section( ".", 0, 0 );
1742  dindex = QString().sprintf( "%4.4d", kndx++ );
1743  ddGUID = cdesc.dataGUID;
1744  dpGUID = cdesc.parentGUID;
1745  dsorts = dlabel + ":" + dindex + ":" + ddGUID + ":" + dpGUID;
1746 
1747  sortm << dsorts;
1748  orphm << dsorts;
1749  guidsm << ddGUID;
1750  tdess << cdesc;
1751 DbgLv(2) << "N orphan:" << orphn.at( ii ) << ii;
1752 DbgLv(2) << " M dummy:" << dsorts;
1753  progress->setValue( ++istep );
1754  qApp->processEvents();
1755  }
1756 DbgLv(2) << "(2) orphan: N size M size" << orphn.size() << orphm.size();
1757 DbgLv(2) << "SrtD: Orph(N)" << nowTime();
1758 
1759  ndmy = 0;
1760 
1761  for ( int ii = 0; ii < orphm.size(); ii++ )
1762  { // for each orphan model, create a dummy edit
1763  dsorts = orphm.at( ii );
1764  dlabel = dsorts.section( ":", 0, 0 );
1765  dindex = dsorts.section( ":", 1, 1 );
1766  ddGUID = dsorts.section( ":", 2, 2 ).simplified();
1767  dpGUID = dsorts.section( ":", 3, 3 ).simplified();
1768  jndx = dindex.toInt();
1769  cdesc = tdess[ jndx ];
1770 
1771  if ( dpGUID.length() < 16 || dpGUID == dmyGUID )
1772  { // handle case where there is no valid parentGUID
1773  if ( ndmy == 0 ) // first time: create one
1774  dpGUID = dmyGUID;
1775  else
1776  dpGUID = ppGUID; // afterwards: re-use same parent
1777 
1778  kk = sortm.indexOf( dsorts ); // find index in full list
1779  dsorts = dlabel + ":" + dindex + ":" + ddGUID + ":" + dpGUID;
1780 
1781  if ( kk >= 0 )
1782  { // replace present record for new parentGUID
1783  sortm.replace( kk, dsorts );
1784  cdesc.parentGUID = dpGUID;
1785  tdess[ jndx ] = cdesc;
1786  }
1787 
1788  if ( ndmy > 0 ) // after 1st time, skip creating new parent
1789  continue;
1790 
1791  ndmy++; // flag that we have a parent for invalid ones
1792  ppGUID = dpGUID; // save the GUID for new dummy parent
1793  }
1794 
1795  // If this record is no longer an orphan, skip creating new parent
1796  if ( guidse.indexOf( dpGUID ) >= 0 )
1797  continue;
1798 
1799  if ( dpGUID == dmyGUID )
1800  cdesc.label = "Dummy-Edit-for-Orphans";
1801 
1802  cdesc.parentID = cdesc.recordID;
1803  cdesc.recordID = -1;
1804  cdesc.recType = 2;
1805  cdesc.subType = "";
1806  cdesc.recState = NOSTAT;
1807  cdesc.dataGUID = dpGUID;
1808  cdesc.parentGUID = dmyGUID;
1809  cdesc.parentID = -1;
1810  cdesc.filename = "";
1811  cdesc.contents = "";
1812  cdesc.label = cdesc.label.section( ".", 0, 0 );
1813  cdesc.description = cdesc.label + "--ARTIFICIAL-RECORD";
1814  cdesc.lastmodDate = US_Util::toUTCDatetimeText(
1815  QDateTime::currentDateTime().toUTC()
1816  .toString( Qt::ISODate ), true );
1817  cdesc.filemodDate = cdesc.lastmodDate;
1818 
1819  dlabel = dlabel.section( ".", 0, 0 );
1820  dindex = QString().sprintf( "%4.4d", kndx++ );
1821  ddGUID = cdesc.dataGUID;
1822  dpGUID = cdesc.parentGUID;
1823  dsorts = dlabel + ":" + dindex + ":" + ddGUID + ":" + dpGUID;
1824 
1825  sorte << dsorts;
1826  orphe << dsorts;
1827  guidse << ddGUID;
1828  tdess << cdesc;
1829 DbgLv(2) << "M orphan:" << orphm.at( ii ) << ii;
1830 DbgLv(2) << " E dummy:" << dsorts;
1831  progress->setValue( ++istep );
1832  qApp->processEvents();
1833  }
1834 
1835 DbgLv(2) << "(3) orphan: N size M size" << orphn.size() << orphm.size();
1836 DbgLv(2) << "SrtD: Orph(M)" << nowTime();
1837  ndmy = 0;
1838 DbgLv(2) << "(4) orphan: M size E size" << orphm.size() << orphe.size();
1839 
1840  for ( int ii = 0; ii < orphe.size(); ii++ )
1841  { // for each orphan edit, create a dummy raw
1842  dsorts = orphe.at( ii );
1843  dlabel = dsorts.section( ":", 0, 0 );
1844  dindex = dsorts.section( ":", 1, 1 );
1845  ddGUID = dsorts.section( ":", 2, 2 ).simplified();
1846  dpGUID = dsorts.section( ":", 3, 3 ).simplified();
1847  jndx = dindex.toInt();
1848  cdesc = tdess[ jndx ];
1849 
1850  if ( dpGUID.length() < 2 )
1851  { // handle case where there is no valid parentGUID
1852  if ( ndmy == 0 ) // first time: create one
1853  dpGUID = dmyGUID;
1854  else
1855  dpGUID = ppGUID; // afterwards: re-use same parent
1856 
1857  kk = sorte.indexOf( dsorts ); // find index in full list
1858  dsorts = dlabel + ":" + dindex + ":" + ddGUID + ":" + dpGUID;
1859 
1860  if ( kk >= 0 )
1861  { // replace present record for new parentGUID
1862  sorte.replace( kk, dsorts );
1863  cdesc.parentGUID = dpGUID;
1864  tdess[ jndx ] = cdesc;
1865  }
1866 
1867  if ( ndmy > 0 ) // after 1st time, skip creating new parent
1868  continue;
1869 
1870  ndmy++; // flag that we have a parent for invalid ones
1871  ppGUID = dpGUID; // save the GUID for new dummy parent
1872  }
1873 
1874  // If this record is no longer an orphan, skip creating new parent
1875  if ( guidsr.indexOf( dpGUID ) >= 0 )
1876  continue;
1877 
1878  if ( dpGUID == dmyGUID )
1879  cdesc.label = "Dummy-Raw-for-Orphans";
1880 
1881  cdesc.parentID = cdesc.recordID;
1882  cdesc.recordID = -1;
1883  cdesc.recType = 1;
1884  cdesc.subType = "";
1885  cdesc.recState = NOSTAT;
1886  cdesc.dataGUID = dpGUID;
1887  cdesc.parentGUID = dmyGUID;
1888  cdesc.parentID = -1;
1889  cdesc.filename = "";
1890  cdesc.contents = "";
1891  cdesc.label = cdesc.label.section( ".", 0, 0 );
1892  cdesc.description = cdesc.label + "--ARTIFICIAL-RECORD";
1893  cdesc.lastmodDate = US_Util::toUTCDatetimeText(
1894  QDateTime::currentDateTime().toUTC()
1895  .toString( Qt::ISODate ), true );
1896  cdesc.filemodDate = cdesc.lastmodDate;
1897 
1898  dlabel = dlabel.section( ".", 0, 0 );
1899  dindex = QString().sprintf( "%4.4d", kndx++ );
1900  ddGUID = cdesc.dataGUID;
1901  dpGUID = cdesc.parentGUID;
1902  dsorts = dlabel + ":" + dindex + ":" + ddGUID + ":" + dpGUID;
1903 
1904  sortr << dsorts;
1905  guidsr << ddGUID;
1906  tdess << cdesc;
1907 DbgLv(2) << "E orphan:" << orphe.at( ii );
1908 DbgLv(2) << " R dummy:" << dsorts;
1909  progress->setValue( ++istep );
1910  qApp->processEvents();
1911  }
1912 DbgLv(2) << "(5) orphan: M size E size" << orphm.size() << orphe.size();
1913 DbgLv(2) << "SrtD: Orph(E)" << nowTime();
1914 
1915 //for ( int ii = 0; ii < sortr.size(); ii++ )
1916 // DbgLv(2) << "R entry:" << sortr.at( ii );
1917  int countR = sortr.size(); // count of each kind in sorted lists
1918  int countE = sorte.size();
1919  int countM = sortm.size();
1920  int countN = sortn.size();
1921 
1922  //sortr.sort(); // re-sort for dummy additions
1923  //sorte.sort();
1924  //sortm.sort();
1925  //sortn.sort();
1926 DbgLv(1) << "sort/dumy: count REMN" << countR << countE << countM << countN;
1927 //for(int ii=0;ii<countM;ii++) DbgLv(2) << "sm" << ii << "++ " << sortm[ii];
1928 
1929  int noutR = 0; // count of each kind in hierarchical output
1930  int noutE = 0;
1931  int noutM = 0;
1932  int noutN = 0;
1933  int indx;
1934  int pstate = REC_LO | PAR_LO;
1935 
1936  descs.clear(); // reset input vector to become sorted output
1937  lb_status->setText( tr( "Building Sorted Trees..." ) );
1938  istep = 0;
1939  progress->setMaximum( countR );
1940  progress->setValue ( istep );
1941  qApp->processEvents();
1942 
1943  // rebuild the description vector with sorted trees
1944  for ( int ii = 0; ii < countR; ii++ )
1945  { // loop to output sorted Raw records
1946  QString recr = sortr[ ii ];
1947  QString didr = recr.section( ":", 2, 2 );
1948  QString pidr = recr.section( ":", 3, 3 );
1949  indx = recr.section( ":", 1, 1 ).toInt();
1950  cdesc = tdess.at( indx );
1951 
1952  // set up a default parent state flag
1953  pstate = cdesc.recState;
1954  pstate = ( pstate & REC_DB ) != 0 ? ( pstate | PAR_DB ) : pstate;
1955  pstate = ( pstate & REC_LO ) != 0 ? ( pstate | PAR_LO ) : pstate;
1956 
1957  // new state is the default, or NOSTAT if this is a dummy record
1958  cdesc.recState = record_state_flag( cdesc, pstate );
1959 
1960  descs << cdesc; // output Raw rec
1961  noutR++;
1962 
1963  // set up parent state for children to follow
1964  int rpstate = cdesc.recState;
1965 
1966  for ( int jj = 0; jj < countE; jj++ )
1967  { // loop to output sorted Edit records for the above Raw
1968  QString rece = sorte[ jj ];
1969  QString pide = rece.section( ":", 3, 3 );
1970 
1971  if ( pide != didr ) // skip if current Raw not parent
1972  continue;
1973 
1974  QString dide = rece.section( ":", 2, 2 );
1975  indx = rece.section( ":", 1, 1 ).toInt();
1976  cdesc = tdess.at( indx );
1977  cdesc.recState = record_state_flag( cdesc, rpstate );
1978 
1979  descs << cdesc; // output Edit rec
1980  noutE++;
1981 
1982  // set up parent state for children to follow
1983  int epstate = cdesc.recState;
1984 
1985  for ( int mm = 0; mm < countM; mm++ )
1986  { // loop to output sorted Model records for above Edit
1987  QString recm = sortm[ mm ];
1988  QString pidm = recm.section( ":", 3, 3 );
1989 
1990  if ( pidm != dide ) // skip if current Edit not parent
1991  continue;
1992 
1993  QString didm = recm.section( ":", 2, 2 );
1994  indx = recm.section( ":", 1, 1 ).toInt();
1995  cdesc = tdess.at( indx );
1996  cdesc.recState = record_state_flag( cdesc, epstate );
1997 
1998  descs << cdesc; // output Model rec
1999 
2000  noutM++;
2001 
2002  // set up parent state for children to follow
2003  int mpstate = cdesc.recState;
2004 
2005  for ( int nn = 0; nn < countN; nn++ )
2006  { // loop to output sorted Noise records for above Model
2007  QString recn = sortn[ nn ];
2008  QString pidn = recn.section( ":", 3, 3 );
2009 
2010  if ( pidn != didm ) // skip if current Model not parent
2011  continue;
2012 
2013  indx = recn.section( ":", 1, 1 ).toInt();
2014  cdesc = tdess.at( indx );
2015  cdesc.recState = record_state_flag( cdesc, mpstate );
2016 
2017  descs << cdesc; // output Noise rec
2018 
2019  noutN++;
2020  }
2021  }
2022  }
2023  progress->setValue( ++istep );
2024  qApp->processEvents();
2025  }
2026 DbgLv(2) << "SrtD: END" << nowTime();
2027 
2028  if ( noutR != countR || noutE != countE ||
2029  noutM != countM || noutN != countN )
2030  { // not all accounted for, so we will need some dummy parents
2031  DbgLv(1) << "sort_desc: count REMN"
2032  << countR << countE << countM << countN;
2033  DbgLv(1) << "sort_desc: nout REMN"
2034  << noutR << noutE << noutM << noutN;
2035  }
2036 }
2037 
2038 // review sorted string lists for duplicate GUIDs
2039 bool US_DataModel::review_descs( QStringList& sorts,
2040  QVector< DataDesc >& descv )
2041 {
2042  bool abort = false;
2043  int nrecs = sorts.size();
2044  int nmult = 0;
2045  int kmult = 0;
2046  int ityp;
2047  QString cGUID;
2048  QString pGUID;
2049  QString rtyp;
2050  QVector< int > multis;
2051  const char* rtyps[] = { "RawData", "EditedData", "Model", "Noise" };
2052  QStringList tGUIDs;
2053 
2054  if ( nrecs < 1 )
2055  return abort;
2056 
2057  int ii = sorts[ 0 ].section( ":", 1, 1 ).toInt();
2058  ityp = descv[ ii ].recType;
2059  rtyp = QString( rtyps[ ityp - 1 ] );
2060 
2061  if ( descv[ ii ].recordID >= 0 )
2062  rtyp = "DB " + rtyp;
2063  else
2064  rtyp = "Local " + rtyp;
2065 DbgLv(2) << "RvwD: ii ityp rtyp nrecs" << ii << ityp << rtyp << nrecs;
2066  cGUID = sorts[ 0 ].section( ":", 2, 2 );
2067  tGUIDs << cGUID;
2068 
2069  for ( int ii = 1; ii < nrecs; ii++ )
2070  { // do a pass to determine if there are duplicate GUIDs
2071  cGUID = sorts[ ii ].section( ":", 2, 2 ); // current rec GUID
2072  kmult = 0; // flag no multiples yet
2073 
2074  int jj = tGUIDs.indexOf( cGUID );
2075 
2076  if ( jj >= 0 )
2077  {
2078  kmult++;
2079  if ( ! multis.contains( jj ) )
2080  { // not yet marked, so mark previous as multiple
2081  multis << jj; // save index
2082  nmult++; // bump count
2083  }
2084  }
2085 
2086  if ( kmult > 0 )
2087  { // this pass found a duplicate: save the index and bump count
2088  multis << ii;
2089  nmult++;
2090 DbgLv(1) << "RvwD: nmult" << nmult << "ii,jj" << ii << jj
2091  << "cGUID,pGUID" << cGUID << tGUIDs[jj];
2092  }
2093 
2094  tGUIDs << cGUID;
2095 //DbgLv(2) << "RvwD: ii kmult nmult" << ii << kmult << nmult;
2096  }
2097 
2098 DbgLv(2) << "RvwD: GUID nmult" << nmult << nowTime();
2099  if ( nmult > 0 )
2100  { // there were multiple instances of the same GUID
2101  QMessageBox msgBox;
2102  QString msg;
2103 
2104  // format a message for the warning pop-up
2105  msg =
2106  tr( "There are %1 %2 records that have\n" ).arg( nmult ).arg( rtyp ) +
2107  tr( "the same GUID as another.\n" ) +
2108  tr( "You should correct the situation before proceeding.\n" ) +
2109  tr( " Click \"Ok\" to see details, then abort.\n" ) +
2110  tr( " Click \"Ignore\" to proceed to further review.\n" );
2111  msgBox.setWindowTitle( tr( "Duplicate %1 Records" ).arg( rtyp ) );
2112  msgBox.setText( msg );
2113  msgBox.setStandardButtons( QMessageBox::Ok | QMessageBox::Ignore );
2114  msgBox.setDefaultButton( QMessageBox::Ok );
2115 
2116  if ( msgBox.exec() == QMessageBox::Ok )
2117  { // user wants details, so display them
2118  QString fileexts = tr( "Text,Log files (*.txt *.log);;" )
2119  + tr( "All files (*)" );
2120  QString pGUID = "";
2121  QString cGUID;
2122  QString label;
2123 
2124  msg =
2125  tr( "Review the details below on duplicate records.\n" ) +
2126  tr( "Save or Print the contents of this message.\n" ) +
2127  tr( "Decide which of the duplicates should be removed.\n" ) +
2128  tr( "Close the main US_DataModel window after exiting here.\n" ) +
2129  tr( "\nSummary of Duplicates:\n\n" );
2130 
2131  for ( int ii = 0; ii < nmult; ii++ )
2132  { // add summary lines on duplicates
2133  int jj = multis.at( ii );
2134  cGUID = sorts.at( jj ).section( ":", 2, 2 );
2135  label = sorts.at( jj ).section( ":", 0, 0 );
2136 
2137  if ( cGUID != pGUID )
2138  { // first instance of this GUID: show GUID
2139  msg += tr( "GUID: " ) + cGUID + "\n";
2140  pGUID = cGUID;
2141  }
2142 
2143  // one label line for each multiple
2144  msg += tr( " Label: " ) + label + "\n";
2145  }
2146 
2147  msg += tr( "\nDetails of Duplicates:\n\n" );
2148 
2149  for ( int ii = 0; ii < nmult; ii++ )
2150  { // add detail lines
2151  int jj = multis.at( ii );
2152  cGUID = sorts.at( jj ).section( ":", 2, 2 );
2153  pGUID = sorts.at( jj ).section( ":", 3, 3 );
2154  label = sorts.at( jj ).section( ":", 0, 0 );
2155  int kk = sorts.at( jj ).section( ":", 1, 1 ).toInt();
2156  cdesc = descv[ kk ];
2157 
2158  msg += tr( "GUID: " ) + cGUID + "\n" +
2159  tr( " ParentGUID: " ) + pGUID + "\n" +
2160  tr( " Label: " ) + label + "\n" +
2161  tr( " Description: " ) + cdesc.description + "\n" +
2162  tr( " DB record ID: %1" ).arg( cdesc.recordID ) + "\n" +
2163  tr( " File Directory: " ) +
2164  cdesc.filename.section( "/", 0, -2 ) + "\n" +
2165  tr( " File Name: " ) +
2166  cdesc.filename.section( "/", -1, -1 ) + "\n" +
2167  tr( " File Last Mod: " ) +
2168  cdesc.filemodDate + "\n" +
2169  tr( " Last Mod Date: " ) +
2170  cdesc.lastmodDate + "\n";
2171  }
2172 
2173  // pop up text dialog
2174  US_Editor* editd = new US_Editor( US_Editor::LOAD, true, fileexts );
2175  editd->setWindowTitle( tr( "Data Set Duplicate GUID Details" ) );
2176  editd->move( QCursor::pos() + QPoint( 200, 200 ) );
2177  editd->resize( 600, 500 );
2178  editd->e->setFont( QFont( US_Widgets::fixedFont().family(),
2179  US_GuiSettings::fontSize() ) );
2180  editd->e->setText( msg );
2181  editd->show();
2182 
2183  abort = true; // tell caller to abort data tree build
2184  }
2185 
2186  else
2187  {
2188  abort = false; // signal to proceed with data tree build
2189  }
2190 DbgLv(2) << "review_descs abort flag:" << abort;
2191  }
2192 
2193  return abort;
2194 }
2195 
2196 // find index of substring at given position in strings of string list
2197 int US_DataModel::index_substring( QString ss, int ixs, QStringList& sl )
2198 {
2199  QString sexp = "XXX";
2200  QRegExp rexp;
2201 
2202  if ( ixs == 0 )
2203  sexp = ss + ":*"; // label at beginning of strings in list
2204 
2205  else if ( ixs == 1 || ixs == 2 )
2206  sexp = "*:" + ss + ":*"; // RecIndex/recGUID in middle of list strings
2207 
2208  else if ( ixs == 3 )
2209  sexp = "*:" + ss; // parentGUID at end of strings in list
2210 
2211  rexp = QRegExp( sexp, Qt::CaseSensitive, QRegExp::Wildcard );
2212 
2213  return sl.indexOf( rexp );
2214 }
2215 
2216 // Get sublist from string list of substring matches at given string position
2217 QStringList US_DataModel::filter_substring( QString ss, int ixs,
2218  QStringList& sl )
2219 {
2220  QStringList subl;
2221 
2222  if ( ixs == 0 )
2223  // match label at beginning of strings in list
2224  subl = sl.filter( QRegExp( "^" + ss + ":" ) );
2225 
2226  else if ( ixs == 1 || ixs == 2 )
2227  // match RecIndex or recGUID in middle of strings in list
2228  subl = sl.filter( ":" + ss + ":" );
2229 
2230  else if ( ixs == 3 )
2231  // match parentGUID at end of strings in list
2232  subl = sl.filter( QRegExp( ":" + ss + "$" ) );
2233 
2234  return subl;
2235 }
2236 
2237 // List orphans of a record type (in rec list, no tie to parent list)
2238 QStringList US_DataModel::list_orphans( QStringList& rlist,
2239  QStringList& plist )
2240 {
2241  QStringList olist;
2242  QStringList tlist;
2243 
2244  for ( int ii = 0; ii < plist.size(); ii++ )
2245  { // Build test-parent-GUID list
2246  tlist << plist.at( ii ).section( ":", 2, 2 );
2247  }
2248 
2249  for ( int ii = 0; ii < rlist.size(); ii++ )
2250  { // examine parentGUID for each record in the list
2251  QString pReco = rlist.at( ii );
2252  QString pGUID = pReco.section( ":", 3, 3 );
2253 
2254  // see if it is the recordGUID of any in the potential parent list
2255  if ( ! tlist.contains( pGUID ) )
2256  olist << pReco; // no parent found, so add to the orphan list
2257  }
2258 
2259  return olist;
2260 }
2261 
2262 // return a record state flag with parent state ORed in
2263 int US_DataModel::record_state_flag( DataDesc descr, int pstate )
2264 {
2265  int state = descr.recState;
2266 
2267  if ( descr.recState == NOSTAT ||
2268  descr.description.contains( "-ARTIFICIAL" ) )
2269  state = NOSTAT; // mark a dummy record
2270 
2271  else
2272  { // detect and mark parentage of non-dummy
2273  if ( ( pstate & REC_DB ) != 0 )
2274  state = state | PAR_DB; // mark a record with db parent
2275 
2276  if ( ( pstate & REC_LO ) != 0 )
2277  state = state | PAR_LO; // mark a record with local parent
2278  }
2279 
2280  return state;
2281 }
2282 
2283 // compose concatenation on which to sort (label:index:dataGUID:parentGUID)
2284 QString US_DataModel::sort_string( DataDesc ddesc, int indx )
2285 { // create string for ascending sort on label
2286  QString label = ( ddesc.recType < 3 ) ? ddesc.label : ddesc.description;
2287  int lablen = label.length();
2288  if ( lablen < maxdlen )
2289  label = label.leftJustified( maxdlen, ' ' );
2290 
2291  QString ostr = label // label to sort on
2292  + ":" + QString().sprintf( "%4.4d", indx ) // index in desc. vector
2293  + ":" + ddesc.dataGUID // data GUID
2294  + ":" + ddesc.parentGUID; // parent GUID
2295  return ostr;
2296 }
2297 
2298 // compose string describing model type
2299 QString US_DataModel::model_type( int imtype, int nassoc, int gtype, bool isMC )
2300 {
2301  QString mtype;
2302 
2303  // format the base model type string
2304  switch ( imtype )
2305  {
2306  default:
2307  case (int)US_Model::MANUAL:
2308  mtype = "MANUAL";
2309  break;
2310  case (int)US_Model::TWODSA:
2311  mtype = "2DSA";
2312  break;
2313  case (int)US_Model::TWODSA_MW:
2314  mtype = "2DSA-MW";
2315  break;
2316  case (int)US_Model::GA:
2317  mtype = "GA";
2318  break;
2319  case (int)US_Model::GA_MW:
2320  mtype = "GA-MW";
2321  break;
2322  case (int)US_Model::COFS:
2323  mtype = "COFS";
2324  break;
2325  case (int)US_Model::FE:
2326  mtype = "FE";
2327  break;
2328  case (int)US_Model::PCSA:
2329  mtype = "PCSA";
2330  break;
2331  case (int)US_Model::CUSTOMGRID:
2332  mtype = "CUSTOMGRID";
2333  break;
2334  }
2335 
2336  // add RA for Reversible Associations (if associations count > 1)
2337  if ( nassoc > 1 )
2338  mtype = mtype + "-RA";
2339 
2340  // add FM | GL | SG for Fit-Meniscus|GLobal|SuperGlobal
2341  if ( gtype == (int)US_Model::MENISCUS )
2342  mtype = mtype + "-FM";
2343 
2344  else if ( gtype == (int)US_Model::GLOBAL )
2345  mtype = mtype + "-GL";
2346 
2347  else if ( gtype == (int)US_Model::SUPERGLOBAL )
2348  mtype = mtype + "-SG";
2349 
2350  // add MC for Monte Carlo
2351  if ( isMC )
2352  mtype = mtype + "-MC";
2353 
2354  return mtype;
2355 }
2356 
2357 // compose string describing model type
2358 QString US_DataModel::model_type( US_Model model )
2359 {
2360  // return model type string based on flags in the model object
2361  return model_type( (int)model.analysis, model.associations.size(),
2362  (int)model.global, model.monteCarlo );
2363 }
2364 
2365 // compose string describing model type
2366 QString US_DataModel::model_type( QString modxml )
2367 {
2368  QChar quo( '"' );
2369  int jj;
2370  int imtype;
2371  int nassoc;
2372  int gtype;
2373  bool isMC;
2374 
2375  // model type number from type attribute
2376  jj = modxml.indexOf( " analysisType=" );
2377  imtype = ( jj < 1 ) ? 0 : modxml.mid( jj ).section( quo, 1, 1 ).toInt();
2378 
2379  // count of associations is count of K_d attributes present
2380  nassoc = modxml.count( "K_d=" );
2381  nassoc = ( nassoc == 0 ) ? modxml.count( "k_assoc=" ) : nassoc;
2382  nassoc = ( nassoc == 0 ) ? modxml.count( "k_eq=" ) : nassoc;
2383 
2384  // global type number from type attribute
2385  jj = modxml.indexOf( " globalType=" );
2386  gtype = ( jj < 1 ) ? 0 : modxml.mid( jj ).section( quo, 1, 1 ).toInt();
2387 
2388  // flag if MonteCarlo
2389  jj = modxml.indexOf( " MonteCarlo=\"1" );
2390  isMC = ( jj > 0 );
2391 
2392  // return model type string based on integer flags
2393  return model_type( imtype, nassoc, gtype, isMC );
2394 }
2395 
2396 void US_DataModel::dummy_data()
2397 {
2398  adescs.clear();
2399  ddescs.clear();
2400  ldescs.clear();
2401 
2402  cdesc.recType = 1;
2403  cdesc.recState = REC_DB | PAR_DB;
2404  cdesc.subType = "";
2405  cdesc.label = "item_1_2";
2406  cdesc.description = "demo1_veloc";
2407  cdesc.dataGUID = "demo1_veloc";
2408  cdesc.parentID = 1;
2409  cdesc.recordID = 1;
2410  cdesc.filename = "";
2411  adescs<<cdesc;
2412  ddescs<<cdesc;
2413 
2414  cdesc.recType = 2;
2415  cdesc.recState = REC_DB | REC_LO | PAR_DB | PAR_LO;
2416  cdesc.subType = "RA";
2417  cdesc.label = "item_2_2";
2418  cdesc.description = "demo1_veloc";
2419  cdesc.contents = "AA 12 AA 12";
2420  cdesc.parentID = 1;
2421  cdesc.recordID = 2;
2422  cdesc.filename = "demo1_veloc_edit.xml";
2423  adescs<<cdesc;
2424  ddescs<<cdesc;
2425  ldescs<<cdesc;
2426 
2427  cdesc.recType = 3;
2428  cdesc.recState = REC_LO | PAR_LO;
2429  cdesc.subType = "2DSA";
2430  cdesc.label = "item_3_2";
2431  cdesc.description = "demo1_veloc.sa2d.model.11";
2432  cdesc.parentID = 2;
2433  cdesc.recordID = -1;
2434  cdesc.filename = "demo1_veloc_model.xml";
2435  adescs<<cdesc;
2436  ldescs<<cdesc;
2437 
2438  cdesc.recType = 4;
2439  cdesc.recState = REC_DB | REC_LO | PAR_DB | PAR_LO;
2440  cdesc.subType = "TI";
2441  cdesc.label = "item_4_2";
2442  cdesc.description = "demo1_veloc.ti_noise";
2443  cdesc.contents = "BB 12 AA 13";
2444  cdesc.parentID = 2;
2445  cdesc.recordID = 3;
2446  cdesc.filename = "demo1_veloc_noise.xml";
2447  adescs<<cdesc;
2448  ddescs<<cdesc;
2449  ldescs<<cdesc;
2450 
2451  cdesc.recType = 2;
2452  cdesc.recState = NOSTAT;
2453  cdesc.subType = "RA";
2454  cdesc.label = "item_5_2";
2455  cdesc.description = "demo1_veloc";
2456  cdesc.contents = "CC 15";
2457  cdesc.recordID = -1;
2458  cdesc.filename = "";
2459  adescs<<cdesc;
2460 }
2461 
2462 QString US_DataModel::expGUIDauc( QString aucfile )
2463 {
2464  QString expGUID = "00000000-0000-0000-0000-000000000000";
2465  QString expfnam = aucfile.section( "/", -1, -1 )
2466  .section( ".", 0, 1 ) + ".xml";
2467  QString expfile = aucfile.section( "/", 0, -2 ) + "/" + expfnam;
2468 
2469  QFile file( expfile );
2470 
2471  if ( file.open( QIODevice::ReadOnly | QIODevice::Text ) )
2472  {
2473  QXmlStreamReader xml( &file );
2474 
2475  while( ! xml.atEnd() )
2476  {
2477  xml.readNext();
2478 
2479  if ( xml.isStartElement() && xml.name() == "experiment" )
2480  {
2481  QXmlStreamAttributes a = xml.attributes();
2482  expGUID = a.value( "guid" ).toString();
2483  break;
2484  }
2485  }
2486 
2487  file.close();
2488  }
2489 
2490  return expGUID;
2491 }
2492 
2493 // Review and reset database description list after changes
2494 void US_DataModel::review_dbase()
2495 {
2496  ddescs.clear(); // Wipe out the old DB descriptions list
2497 DbgLv(1) << "RvDB: #chgr #adesc" << chgrows.size() << adescs.size();
2498 
2499  for ( int row = 0; row < adescs.size(); row++ )
2500  { // Review all records looking for changed DB records
2501  cdesc = adescs[ row ];
2502 
2503  if ( chgrows.contains( row ) )
2504  { // Test whether to add to list if a changed row
2505 DbgLv(1) << "RvDB: row state ID" << row << cdesc.recState << cdesc.recordID;
2506 
2507  if ( cdesc.recordID < 0 ) // skip if DB rec removed
2508  continue; // or local-only
2509 
2510 DbgLv(1) << "RvDB: ++ upd cdesc, row state" << row << cdesc.recState;
2511  }
2512 
2513  else if ( ( cdesc.recState & PAR_DB ) == 0 )
2514  continue; // skip if local-only
2515 
2516  // Modify contents and state to be DB-only
2517  QString contdb = cdesc.contents;
2518  contdb = contdb.section( " ", 0, 0 ) + " " +
2519  contdb.section( " ", 1, 1 );
2520  cdesc.contents = contdb;
2521  cdesc.recState = ( cdesc.recState & PAR_DB ) | REC_DB;
2522 
2523  // Save this DB record to the DB list
2524  ddescs << cdesc;
2525  }
2526 DbgLv(1) << "RvDB: #ddescs" << ddescs.size();
2527 
2528  chgrows.clear(); // Clear changed-row list
2529  lb_status->setText( tr( "Database Review Complete" ) );
2530 }
2531 
Generated on Sun Jun 21 2015 11:40:41 for UltraScan III by   doxygen 1.8.3.1-20130324