14 #include <THnSparse.h>
19 #if defined(__linux__)
21 #elif defined(__APPLE__)
24 #include <sys/types.h>
25 #include <sys/socket.h>
28 #include "NHttpRequest.h"
31 #include <arrow/api.h>
32 #include <arrow/io/api.h>
33 #include <parquet/arrow/reader.h>
34 #include <parquet/exception.h>
52 bool previouslyEnabled = ROOT::IsImplicitMTEnabled();
54 if (ROOT::IsImplicitMTEnabled()) {
55 ROOT::DisableImplicitMT();
64 if (numthreads == -1) {
66 const char * nThreadsEnv = gSystem->Getenv(
"ROOT_MAX_THREADS");
69 numthreads = std::stoul(nThreadsEnv);
71 catch (
const std::exception & e) {
72 NLogError(
"Error parsing ROOT_MAX_THREADS: %s !!! Setting it to '1' ...", e.what());
82 ROOT::EnableThreadSafety();
86 ROOT::EnableImplicitMT(numthreads);
90 if (ROOT::IsImplicitMTEnabled()) {
91 NLogInfo(
"ROOT::ImplicitMT is enabled with number of threads: %d", ROOT::GetThreadPoolSize());
94 return previouslyEnabled;
103 if (filename.find(
"http://") == 0 || filename.find(
"https://") == 0 || filename.find(
"root://") == 0 ||
104 filename.find(
"file://") == 0 || filename.find(
"alien://") == 0) {
107 TString fn(filename.c_str());
108 if (fn.BeginsWith(
"/") || !fn.Contains(
"://")) {
111 NLogError(
"NUtils::IsFileSupported: File '%s' not found", filename.c_str());
120 TString pathStr(gSystem->ExpandPathName(path.c_str()));
122 if (pathStr.BeginsWith(
"http://") || pathStr.BeginsWith(
"https://")) {
127 int http_code = request.
head(pathStr.Data());
128 if (http_code == 200) {
134 else if (pathStr.BeginsWith(
"file://") || pathStr.BeginsWith(
"/") || !pathStr.Contains(
"://")) {
136 return gSystem->AccessPathName(pathStr.Data()) ==
false;
138 else if (pathStr.BeginsWith(
"root://") || pathStr.BeginsWith(
"alien://")) {
140 if (!pathStr.EndsWith(
".root")) {
142 pathStr +=
"?filetype=raw";
144 NLogDebug(
"NUtils::AccessPathName: Trying to open file '%s' ...", pathStr.Data());
145 TFile * f = TFile::Open(pathStr.Data());
146 if (f && !f->IsZombie()) {
155 int NUtils::Cp(std::string source, std::string destination, Bool_t progressbar )
162 if (source.empty()) {
163 NLogError(
"NUtils::Cp: Source file is empty");
166 if (destination.empty()) {
167 NLogError(
"NUtils::Cp: Destination file is empty");
172 NLogError(
"NUtils::Cp: Source file '%s' is not supported", source.c_str());
176 NLogError(
"NUtils::Cp: Destination file '%s' is not supported", destination.c_str());
180 NLogInfo(
"Copying file from '%s' to '%s' ...", source.c_str(), destination.c_str());
181 rc = TFile::Cp(source.c_str(), destination.c_str(), progressbar);
186 const std::vector<std::string> & labels)
191 int nBins = labels.size();
192 TAxis * a =
new TAxis(nBins, 0, nBins);
193 a->SetName(name.c_str());
194 a->SetTitle(title.c_str());
195 for (
int i = 0; i < nBins; i++) {
196 NLogTrace(
"NUtils::CreateAxisFromLabels: Adding label: %s", labels[i].c_str());
197 a->SetBinLabel(i + 1, labels[i].c_str());
203 const std::set<std::string> & labels)
208 int nBins = labels.size();
209 TAxis * a =
new TAxis(nBins, 0, nBins);
210 a->SetName(name.c_str());
211 a->SetTitle(title.c_str());
213 for (
const auto & label : labels) {
214 NLogTrace(
"NUtils::CreateAxisFromLabels: Adding label: %s", label.c_str());
215 a->SetBinLabel(i, label.c_str());
221 THnSparse *
NUtils::Convert(TH1 * h1, std::vector<std::string> names, std::vector<std::string> titles)
228 NLogError(
"TH1 h1 is null");
232 NLogInfo(
"Converting TH1 '%s' to THnSparse ...", h1->GetName());
238 auto bins = std::make_unique<Int_t[]>(nDims);
239 auto xmin = std::make_unique<Double_t[]>(nDims);
240 auto xmax = std::make_unique<Double_t[]>(nDims);
242 TAxis * aIn = h1->GetXaxis();
243 bins[0] = aIn->GetNbins();
244 xmin[0] = aIn->GetXmin();
245 xmax[0] = aIn->GetXmax();
247 THnSparse * hns =
new THnSparseD(h1->GetName(), h1->GetTitle(), nDims, bins.get(), xmin.get(), xmax.get());
250 for (
int i = 0; i < nDims; i++) {
251 TAxis * a = hns->GetAxis(i);
252 TAxis * aIn = h1->GetXaxis();
253 a->SetName(aIn->GetName());
254 a->SetTitle(aIn->GetTitle());
255 if (aIn->GetXbins()->GetSize() > 0) {
257 auto arr = std::make_unique<Double_t[]>(aIn->GetNbins() + 1);
258 arr[0] = aIn->GetBinLowEdge(1);
259 for (
int iBin = 1; iBin <= aIn->GetNbins(); iBin++) {
260 arr[iBin] = aIn->GetBinUpEdge(iBin);
262 a->Set(a->GetNbins(), arr.get());
266 for (
int i = 0; i < nDims; i++) {
267 if (!names[i].empty()) hns->GetAxis(i)->SetName(names[i].c_str());
268 if (!titles[i].empty()) hns->GetAxis(i)->SetTitle(titles[i].c_str());
272 for (Int_t i = 0; i <= h1->GetNbinsX() + 1; i++) {
273 double content = h1->GetBinContent(i);
275 hns->SetBinContent(p, content);
278 hns->SetEntries(h1->GetEntries());
279 if (h1->GetSumw2N() > 0) {
286 THnSparse *
NUtils::Convert(TH2 * h2, std::vector<std::string> names, std::vector<std::string> titles)
292 NLogError(
"TH2 h2 is null");
295 NLogInfo(
"Converting TH2 '%s' to THnSparse ...", h2->GetName());
297 auto bins = std::make_unique<Int_t[]>(nDims);
298 auto xmin = std::make_unique<Double_t[]>(nDims);
299 auto xmax = std::make_unique<Double_t[]>(nDims);
301 for (
int i = 0; i < nDims; i++) {
302 TAxis * aIn =
nullptr;
304 aIn = h2->GetXaxis();
306 aIn = h2->GetYaxis();
308 NLogError(
"Invalid axis index %d", i);
311 bins[i] = aIn->GetNbins();
312 xmin[i] = aIn->GetXmin();
313 xmax[i] = aIn->GetXmax();
316 THnSparse * hns =
new THnSparseD(h2->GetName(), h2->GetTitle(), nDims, bins.get(), xmin.get(), xmax.get());
318 for (Int_t i = 0; i < nDims; i++) {
319 TAxis * a = hns->GetAxis(i);
320 TAxis * aIn =
nullptr;
322 aIn = h2->GetXaxis();
324 aIn = h2->GetYaxis();
326 NLogError(
"Invalid axis index %d", i);
330 a->SetName(aIn->GetName());
331 a->SetTitle(aIn->GetTitle());
332 if (aIn->GetXbins()->GetSize() > 0) {
333 auto arr = std::make_unique<Double_t[]>(aIn->GetNbins() + 1);
334 arr[0] = aIn->GetBinLowEdge(1);
335 for (
int iBin = 1; iBin <= aIn->GetNbins(); iBin++) {
336 arr[iBin] = aIn->GetBinUpEdge(iBin);
338 a->Set(a->GetNbins(), arr.get());
342 for (Int_t i = 0; i < nDims; i++) {
343 if (!names[i].empty()) hns->GetAxis(i)->SetName(names[i].c_str());
344 if (!titles[i].empty()) hns->GetAxis(i)->SetTitle(titles[i].c_str());
348 for (Int_t i = 0; i <= h2->GetNbinsX() + 1; i++) {
349 for (Int_t j = 0; j <= h2->GetNbinsY() + 1; j++) {
350 double content = h2->GetBinContent(i, j);
352 hns->SetBinContent(p, content);
356 hns->SetEntries(h2->GetEntries());
357 if (h2->GetSumw2N() > 0) {
364 THnSparse *
NUtils::Convert(TH3 * h3, std::vector<std::string> names, std::vector<std::string> titles)
371 NLogError(
"TH3 h3 is null");
375 NLogInfo(
"Converting TH3 '%s' to THnSparse ...", h3->GetName());
378 auto bins = std::make_unique<Int_t[]>(nDims);
379 auto xmin = std::make_unique<Double_t[]>(nDims);
380 auto xmax = std::make_unique<Double_t[]>(nDims);
382 for (
int i = 0; i < nDims; i++) {
383 TAxis * aIn =
nullptr;
385 aIn = h3->GetXaxis();
387 aIn = h3->GetYaxis();
389 aIn = h3->GetZaxis();
391 NLogError(
"Invalid axis index %d", i);
394 bins[i] = aIn->GetNbins();
395 xmin[i] = aIn->GetXmin();
396 xmax[i] = aIn->GetXmax();
399 THnSparse * hns =
new THnSparseD(h3->GetName(), h3->GetTitle(), nDims, bins.get(), xmin.get(), xmax.get());
402 for (
int i = 0; i < nDims; i++) {
403 TAxis * a = hns->GetAxis(i);
404 TAxis * aIn =
nullptr;
406 aIn = h3->GetXaxis();
408 aIn = h3->GetYaxis();
410 aIn = h3->GetZaxis();
412 NLogError(
"Invalid axis index %d", i);
416 a->SetName(aIn->GetName());
417 a->SetTitle(aIn->GetTitle());
418 if (aIn->GetXbins()->GetSize() > 0) {
419 auto arr = std::make_unique<Double_t[]>(aIn->GetNbins() + 1);
420 arr[0] = aIn->GetBinLowEdge(1);
421 for (
int iBin = 1; iBin <= aIn->GetNbins(); iBin++) {
422 arr[iBin] = aIn->GetBinUpEdge(iBin);
424 a->Set(a->GetNbins(), arr.get());
428 for (Int_t i = 0; i < nDims; i++) {
429 if (!names[i].empty()) hns->GetAxis(i)->SetName(names[i].c_str());
430 if (!titles[i].empty()) hns->GetAxis(i)->SetTitle(titles[i].c_str());
434 for (Int_t i = 0; i <= h3->GetNbinsX() + 1; i++) {
435 for (Int_t j = 0; j <= h3->GetNbinsY() + 1; j++) {
436 for (Int_t k = 0; k <= h3->GetNbinsZ() + 1; k++) {
437 double content = h3->GetBinContent(i, j, k);
438 Int_t p[3] = {i, j, k};
439 hns->SetBinContent(p, content);
444 hns->SetEntries(h3->GetEntries());
445 if (h3->GetSumw2N() > 0) {
453 std::vector<int> newPoint, Option_t * option)
461 if (hns ==
nullptr) {
462 NLogError(
"NUtils::ReshapeSparseAxes: THnSparse hns is null");
467 NLogTrace(
"NUtils::ReshapeSparseAxes: Order vector is empty");
468 for (
long unsigned int i = 0; i < hns->GetNdimensions() + newAxes.size(); i++) {
469 NLogTrace(
"NUtils::ReshapeSparseAxes: Adding axis %d to order", i);
474 if (order.size() != hns->GetNdimensions() + newAxes.size()) {
475 NLogError(
"NUtils::ReshapeSparseAxes: Invalid size %d [order] != %d [hns->GetNdimensions()+newAxes]", order.size(),
476 hns->GetNdimensions() + newAxes.size());
480 if (newPoint.empty()) {
483 if (newAxes.size() != newPoint.size()) {
484 NLogError(
"NUtils::ReshapeSparseAxes: Invalid size %d [newAxes] != %d [newPoint]", newAxes.size(),
490 for (
size_t i = 0; i < order.size(); i++) {
491 if (order[i] < 0 || order[i] >= hns->GetNdimensions() + (
int)newAxes.size()) {
492 NLogError(
"NUtils::ReshapeSparseAxes: Invalid order[%d]=%d. Value is negative or higher then "
493 "'hns->GetNdimensions() + newAxes.size()' !!!",
500 for (
size_t i = 0; i < order.size(); i++) {
501 for (
size_t j = i + 1; j < order.size(); j++) {
502 if (order[i] == order[j]) {
503 NLogError(
"NUtils::ReshapeSparseAxes: Invalid order[%d]=%d and order[%d]=%d. Value is not unique !!!", i,
504 order[i], j, order[j]);
514 NLogTrace(
"NUtils::ReshapeSparseAxes: Reshaping sparse axes ...");
516 int nDims = hns->GetNdimensions() + newAxes.size();
517 auto bins = std::make_unique<Int_t[]>(nDims);
518 auto xmin = std::make_unique<Double_t[]>(nDims);
519 auto xmax = std::make_unique<Double_t[]>(nDims);
521 int newAxesIndex = 0;
522 for (
int i = 0; i < nDims; i++) {
525 if (id < hns->GetNdimensions()) {
526 a = hns->GetAxis(
id);
527 NLogTrace(
"NUtils::ReshapeSparseAxes: [ORIG] Axis [%d]->[%d]: %s %s %d %.2f %.2f",
id, i, a->GetName(),
528 a->GetTitle(), a->GetNbins(), a->GetXmin(), a->GetXmax());
531 newAxesIndex =
id - hns->GetNdimensions();
532 a = newAxes[newAxesIndex];
533 NLogTrace(
"NUtils::ReshapeSparseAxes: [NEW ] Axis [%d]->[%d]: %s %s %d %.2f %.2f",
id, i, a->GetName(),
534 a->GetTitle(), a->GetNbins(), a->GetXmin(), a->GetXmax());
536 bins[i] = a->GetNbins();
537 xmin[i] = a->GetXmin();
538 xmax[i] = a->GetXmax();
541 THnSparse * hnsNew =
new THnSparseD(hns->GetName(), hns->GetTitle(), nDims, bins.get(), xmin.get(), xmax.get());
544 for (
int i = 0; i < hnsNew->GetNdimensions(); i++) {
545 TAxis * aIn =
nullptr;
546 if (order[i] < hns->GetNdimensions()) {
547 aIn = hns->GetAxis(order[i]);
550 newAxesIndex = order[i] - hns->GetNdimensions();
551 aIn = newAxes[newAxesIndex];
554 TAxis * a = hnsNew->GetAxis(i);
555 a->SetName(aIn->GetName());
556 a->SetTitle(aIn->GetTitle());
557 if (aIn->GetXbins()->GetSize() > 0) {
558 auto arr = std::make_unique<Double_t[]>(aIn->GetNbins() + 1);
559 arr[0] = aIn->GetBinLowEdge(1);
560 for (
int iBin = 1; iBin <= aIn->GetNbins(); iBin++) {
561 arr[iBin] = aIn->GetBinUpEdge(iBin);
563 a->Set(a->GetNbins(), arr.get());
567 if (aIn->IsAlphanumeric()) {
568 for (
int j = 1; j <= aIn->GetNbins(); j++) {
569 const char * label = aIn->GetBinLabel(j);
570 a->SetBinLabel(j, label);
575 if (newPoint.empty()) {
576 NLogTrace(
"NUtils::ReshapeSparseAxes: New point is empty, filling is skipped and doing reset ...");
582 if (hns->GetNbins() > 0) {
584 NLogTrace(
"NUtils::ReshapeSparseAxes: Filling all bins ...");
585 for (Long64_t i = 0; i < hns->GetNbins(); i++) {
586 auto p = std::make_unique<Int_t[]>(nDims);
587 auto pNew = std::make_unique<Int_t[]>(nDims);
588 hns->GetBinContent(i, p.get());
589 Double_t v = hns->GetBinContent(i);
591 for (
int j = 0; j < nDims; j++) {
593 if (id < hns->GetNdimensions()) {
597 newAxesIndex =
id - hns->GetNdimensions();
598 pNew[j] = newPoint[newAxesIndex];
601 hnsNew->SetBinContent(pNew.get(), v);
603 hnsNew->SetEntries(hns->GetEntries());
606 if (opt.Contains(
"E")) {
607 NLogTrace(
"ReshapeSparseAxes: Calculating sumw2 ...");
610 NLogTrace(
"ReshapeSparseAxes: Reshaped sparse axes:");
612 for (
int i = 0; i < nDims; i++) {
613 TAxis * a = hnsNew->GetAxis(i);
614 NLogTrace(
"ReshapeSparseAxes: Axis %d: %s %s %d %.2f %.2f", i, a->GetName(), a->GetTitle(), a->GetNbins(),
615 a->GetXmin(), a->GetXmax());
632 max_val = -std::numeric_limits<double>::max();
633 min_val = std::numeric_limits<double>::max();
635 int first_bin_x = include_overflow_underflow ? 0 : 1;
636 int last_bin_x = include_overflow_underflow ? h->GetNbinsX() + 1 : h->GetNbinsX();
638 int first_bin_y = include_overflow_underflow ? 0 : 1;
639 int last_bin_y = include_overflow_underflow ? h->GetNbinsY() + 1 : h->GetNbinsY();
641 int first_bin_z = include_overflow_underflow ? 0 : 1;
642 int last_bin_z = include_overflow_underflow ? h->GetNbinsZ() + 1 : h->GetNbinsZ();
645 if (h->GetDimension() == 1) {
646 for (
int i = first_bin_x; i <= last_bin_x; ++i) {
647 double content = h->GetBinContent(i);
648 if (content > max_val) max_val = content;
649 if (content < min_val) min_val = content;
652 else if (h->GetDimension() == 2) {
653 for (
int i = first_bin_x; i <= last_bin_x; ++i) {
654 for (
int j = first_bin_y; j <= last_bin_y; ++j) {
655 double content = h->GetBinContent(i, j);
656 if (content > max_val) max_val = content;
657 if (content < min_val) min_val = content;
661 else if (h->GetDimension() == 3) {
662 for (
int i = first_bin_x; i <= last_bin_x; ++i) {
663 for (
int j = first_bin_y; j <= last_bin_y; ++j) {
664 for (
int k = first_bin_z; k <= last_bin_z; ++k) {
665 double content = h->GetBinContent(i, j, k);
666 if (content > max_val) max_val = content;
667 if (content < min_val) min_val = content;
673 NLogWarning(
"GetTrueHistogramMinMax: Histogram '%s' has unsupported dimension %d. "
674 "Using GetMaximum/GetMinimum as fallback.",
675 h->GetName(), h->GetDimension());
677 max_val = h->GetMaximum();
678 min_val = h->GetMinimum();
682 if (max_val == -std::numeric_limits<double>::max() && min_val == std::numeric_limits<double>::max()) {
696 if (path.empty())
return false;
698 TString dir(path.c_str());
699 bool isLocalFile = dir.BeginsWith(
"file://");
701 dir.ReplaceAll(
"file://",
"");
703 isLocalFile = !dir.Contains(
"://");
706 if (!isLocalFile)
return true;
708 std::string pwd = gSystem->pwd();
709 if (dir[0] !=
'/') dir = (pwd +
"/" + std::string(dir.Data())).c_str();
710 dir.ReplaceAll(
"?remote=1&",
"?");
711 dir.ReplaceAll(
"?remote=1",
"");
712 dir.ReplaceAll(
"&remote=1",
"");
713 TUrl url(dir.Data());
715 const std::string localDir = url.GetFile();
716 return gSystem->mkdir(localDir.c_str(), kTRUE) == 0;
725 filename = gSystem->ExpandPathName(filename.c_str());
726 if (createLocalDir) {
727 if (!mode.compare(
"RECREATE") || !mode.compare(
"UPDATE") || !mode.compare(
"WRITE")) {
728 const std::string dir = gSystem->GetDirName(filename.c_str()).Data();
732 return TFile::Open(filename.c_str(), mode.c_str());
742 TFile * f =
OpenFile(TString::Format(
"%s?filetype=raw", filename.c_str()).Data());
749 auto buff = std::make_unique<char[]>(buffsize + 1);
752 Long64_t buffread = 0;
753 while (buffread < f->GetSize()) {
754 if (buffread + buffsize > f->GetSize()) buffsize = f->GetSize() - buffread;
757 f->ReadBuffer(buff.get(), buffread, buffsize);
758 buff[buffsize] =
'\0';
759 content += buff.get();
760 buffread += buffsize;
771 TFile * f =
OpenFile(TString::Format(
"%s?filetype=raw", filename.c_str()).Data(),
"RECREATE");
773 NLogError(
"Error: Problem opening file '%s' in 'rw' mode ...", filename.c_str());
776 f->WriteBuffer(content.c_str(), content.size());
788 if (filename.find(
"http://") == 0 || filename.find(
"https://") == 0) {
790 content = request.
get(filename);
791 if (content.empty()) {
792 Printf(
"Error: Problem fetching macro from '%s' ...", filename.c_str());
798 if (content.empty()) {
799 Printf(
"Error: Problem opening macro '%s' ...", filename.c_str());
803 Printf(
"Using macro '%s' ...", filename.c_str());
804 TUrl url(filename.c_str());
805 std::string basefilename = gSystem->BaseName(url.GetFile());
806 basefilename.pop_back();
807 basefilename.pop_back();
808 TMacro * m =
new TMacro();
809 m->SetName(basefilename.c_str());
810 m->AddLine(content.c_str());
821 if (content.empty()) {
822 NLogError(
"NUtils::LoadJsonFile: Problem opening JSON file '%s' ...", filename.c_str());
827 json myCfg = json::parse(content.c_str());
828 cfg.merge_patch(myCfg);
829 NLogInfo(
"NUtils::LoadJsonFile: Successfully parsed JSON file '%s' ...", filename.c_str());
831 catch (json::parse_error & e) {
832 NLogError(
"NUtils::LoadJsonFile: JSON parse error in file '%s' at byte %d: %s", filename.c_str(), e.byte, e.what());
841 const std::string kPlaceholderBase =
"##RAW_JSON_INJECT_";
844 for (
size_t i = 0; i < injections.size(); ++i) {
845 const auto & keys = injections[i].first;
847 throw std::invalid_argument(
"Keys array must not be empty at injection index " + std::to_string(i));
851 for (
size_t k = 0; k < keys.size() - 1; ++k) {
852 if (!current->contains(keys[k])) {
853 (*current)[keys[k]] = json::object();
855 current = &(*current)[keys[k]];
857 (*current)[keys.back()] = kPlaceholderBase + std::to_string(i) +
"##";
860 std::string result = j.dump();
864 for (
size_t i = 0; i < injections.size(); ++i) {
865 const std::string quotedPlaceholder =
"\"" + kPlaceholderBase + std::to_string(i) +
"##\"";
867 size_t pos = result.find(quotedPlaceholder);
868 if (pos == std::string::npos) {
869 throw std::runtime_error(
"Placeholder not found for key path ending in \"" + injections[i].first.back() +
"\"");
872 while (pos != std::string::npos) {
873 result.replace(pos, quotedPlaceholder.length(), injections[i].second);
874 pos = result.find(quotedPlaceholder, pos + injections[i].second.size());
882 const std::string & injectionsKey)
885 throw std::invalid_argument(
"AddRawJsonInjection: path must not be empty");
891 bool pathReachable =
true;
892 for (
size_t k = 0; k + 1 < path.size(); ++k) {
893 if (!current->is_object() || !current->contains(path[k])) {
894 pathReachable =
false;
897 current = &(*current)[path[k]];
900 std::string valueToStore = rawJson;
901 if (pathReachable && current->is_object() && current->contains(path.back())) {
902 const json & existing = (*current)[path.back()];
903 if (existing.is_object() && !existing.empty()) {
906 size_t lastBrace = valueToStore.rfind(
'}');
907 if (lastBrace != std::string::npos) {
908 std::string extras = existing.dump();
909 std::string extraFields = extras.substr(1, extras.size() - 2);
910 if (!extraFields.empty()) {
911 valueToStore = valueToStore.substr(0, lastBrace) +
"," + extraFields +
"}";
917 if (!j.contains(injectionsKey) || !j[injectionsKey].is_array()) {
918 j[injectionsKey] = json::array();
921 j[injectionsKey].push_back({{
"path", path}, {
"value", valueToStore}});
927 if (!j.contains(injectionsKey) || !j[injectionsKey].is_array()) {
931 for (
const auto & entry : j[injectionsKey]) {
932 if (!entry.contains(
"path") || !entry[
"path"].is_array() || !entry.contains(
"value") || !entry[
"value"].is_string()) {
935 injections.emplace_back(entry[
"path"].get<std::vector<std::string>>(), entry[
"value"].get<std::string>());
938 return !injections.empty();
948 json obj = json::parse(rawJson);
951 for (
const auto & [key, val] : metadata.items()) {
957 catch (
const std::exception & e) {
958 NLogError(
"NUtils::MergeRawJsonWithMetadata: Failed to parse raw JSON: %s", e.what());
963 std::vector<std::string>
NUtils::Find(std::string path, std::string filename)
969 std::vector<std::string> files;
970 TString pathStr = gSystem->ExpandPathName(path.c_str());
971 if (pathStr.IsNull() || filename.empty()) {
972 NLogError(
"NUtils::Find: Path or filename is empty");
976 if (pathStr.BeginsWith(
"root://")) {
977 return FindEos(path, filename);
992 std::vector<std::string> files;
993 if (gSystem->AccessPathName(path.c_str())) {
994 NLogError(
"NUtils::FindLocal: Path '%s' does not exist", path.c_str());
997 NLogInfo(
"Doing find %s -name %s", path.c_str(), filename.c_str());
998 std::string linesMerge =
999 gSystem->GetFromPipe(TString::Format(
"find %s -name %s", path.c_str(), filename.c_str())).Data();
1001 std::stringstream check2(linesMerge);
1003 while (std::getline(check2, line)) {
1004 files.push_back(line);
1014 std::vector<std::string> files;
1015 NLogInfo(
"Doing eos find -f --name %s %s ", filename.c_str(), path.c_str());
1017 TUrl url(path.c_str());
1018 std::string host = url.GetHost();
1019 std::string directory = url.GetFile();
1020 std::string findUrl =
"root://";
1021 findUrl += host +
"//proc/user/";
1022 findUrl +=
"?mgm.cmd=find&mgm.find.match=" + filename;
1023 findUrl +=
"&mgm.path=" + directory;
1024 findUrl +=
"&mgm.format=json&mgm.option=f&filetype=raw";
1025 NLogInfo(
"Doing TFile::Open on '%s' ...", findUrl.c_str());
1028 if (!f)
return files;
1032 int buffsize = 4096;
1034 auto buff = std::make_unique<char[]>(buffsize + 1);
1037 Long64_t buffread = 0;
1038 std::string content;
1039 while (buffread < f->GetSize()) {
1041 if (buffread + buffsize > f->GetSize()) buffsize = f->GetSize() - buffread;
1044 f->ReadBuffer(buff.get(), buffread, buffsize);
1045 buff[buffsize] =
'\0';
1046 content += buff.get();
1047 buffread += buffsize;
1052 std::string ss =
"mgm.proc.stdout=";
1053 size_t pos = ss.size() + 1;
1054 content = content.substr(pos);
1057 std::stringstream check1(content);
1059 std::string intermediate;
1062 std::vector<std::string> tokens;
1063 while (getline(check1, intermediate,
'&')) {
1064 tokens.push_back(intermediate);
1066 std::string linesString = tokens[0];
1068 files.push_back(
"root://" + host +
"/" + line);
1078 std::vector<std::string> out;
1080 size_t end = input.find(delim);
1082 while (end != std::string_view::npos) {
1084 out.emplace_back(input.substr(start, end - start));
1087 end = input.find(delim, start);
1090 if (start < input.length()) {
1091 out.emplace_back(input.substr(start));
1101 std::vector<int> out;
1102 std::vector<std::string> tokens =
Tokenize(input, delim);
1103 for (
auto & t : tokens) {
1104 if (t.empty())
continue;
1105 out.push_back(std::stoi(t));
1111 std::string
NUtils::Join(
const std::vector<std::string> & values,
const char delim)
1118 for (
const auto & v : values) {
1119 if (!out.empty()) out += delim;
1124 std::string
NUtils::Join(
const std::vector<int> & values,
const char delim)
1131 for (
const auto & v : values) {
1132 if (!out.empty()) out += delim;
1133 out += std::to_string(v);
1144 std::vector<std::string> out;
1145 for (
auto & v : values) {
1146 v = std::string(v.begin() + value.size(), v.end());
1152 std::set<std::string>
NUtils::Unique(std::vector<std::string> & paths,
int axis, std::string path,
char token)
1158 std::set<std::string> out;
1160 for (
auto & p : truncatedPaths) {
1161 std::vector<std::string> tokens =
Tokenize(p, token);
1162 out.insert(tokens[axis]);
1172 if (sparse ==
nullptr) {
1173 NLogError(
"Error: Sparse is nullptr ...");
1178 if (axes.size() == 1) {
1179 h = sparse->Projection(axes[0], option);
1181 else if (axes.size() == 2) {
1182 h = sparse->Projection(axes[1], axes[0], option);
1184 else if (axes.size() == 3) {
1185 h = sparse->Projection(axes[0], axes[1], axes[2], option);
1188 NLogError(
"Error: Only projection onto single axis is supported for TH1 ...");
1191 h->SetName(TString::Format(
"%s_proj", sparse->GetName()).Data());
1192 h->SetTitle(TString::Format(
"%s Projection", sparse->GetTitle()).Data());
1196 h->SetDirectory(
nullptr);
1199 for (
size_t i = 0; i < axes.size(); i++) {
1200 TAxis * axisSparse = sparse->GetAxis(axes[i]);
1201 TAxis * axisHist = h->GetXaxis();
1202 if (i == 1) axisHist = h->GetYaxis();
1203 if (i == 2) axisHist = h->GetZaxis();
1205 axisHist->SetName(axisSparse->GetName());
1206 axisHist->SetTitle(axisSparse->GetTitle());
1209 if (axisSparse->IsAlphanumeric()) {
1210 for (
int j = 1; j <= axisSparse->GetNbins(); j++) {
1211 const char * label = axisSparse->GetBinLabel(j);
1212 axisHist->SetBinLabel(j, label);
1221 bool modifyTitle,
bool reset)
1228 if (sparse ==
nullptr) {
1229 NLogError(
"Error: Sparse is nullptr ...");
1232 if (sparse->GetNdimensions() == 0)
return true;
1235 NLogTrace(
"Setting axis ranges on '%s' THnSparse ...", sparse->GetName());
1237 for (
int i = 0; i < sparse->GetNdimensions(); i++) {
1239 NLogTrace(
"Resetting '%s' axis ...", sparse->GetAxis(i)->GetName());
1240 sparse->GetAxis(i)->SetRange(0, 0);
1243 NLogTrace(
"Resetting '%s' axis [%d,%d] ...", sparse->GetAxis(i)->GetName(), 1, sparse->GetAxis(i)->GetNbins());
1244 sparse->GetAxis(i)->SetRange(1, sparse->GetAxis(i)->GetNbins());
1249 if (ranges.empty()) {
1250 NLogTrace(
"No axis ranges to set ...");
1254 TAxis * axis =
nullptr;
1255 TString title = sparse->GetTitle();
1256 if (modifyTitle) title +=
" Ranges:";
1257 for (
size_t i = 0; i < ranges.size(); i++) {
1258 axis = sparse->GetAxis(ranges[i][0]);
1259 NLogTrace(
"Setting axis range %s=[%d,%d] ...", axis->GetName(), ranges[i][1], ranges[i][2]);
1260 if (ranges[i].size() != 3) {
1261 NLogError(
"Error: Axis range must have 3 values, but has %zu ...", ranges[i].size());
1264 axis->SetRange(ranges[i][1], ranges[i][2]);
1265 if (axis->IsAlphanumeric()) {
1267 title += TString::Format(
" %s[%s]", axis->GetName(), axis->GetBinLabel(ranges[i][1]));
1270 title += TString::Format(
" %s[%0.2f - %0.2f]", axis->GetName(), axis->GetBinLowEdge(ranges[i][1]),
1271 axis->GetBinUpEdge(ranges[i][2]));
1274 if (modifyTitle) sparse->SetTitle(title.Data());
1279 bool modifyTitle,
bool reset)
1286 if (sparse ==
nullptr) {
1287 NLogError(
"NUtils::SetAxisRanges: Sparse is nullptr ...");
1290 if (sparse->GetNdimensions() == 0)
return true;
1292 NLogTrace(
"NUtils::SetAxisRanges: Setting axis ranges on '%s' THnSparse ...", sparse->GetName());
1295 for (
int i = 0; i < sparse->GetNdimensions(); i++) {
1297 NLogTrace(
"NUtils::SetAxisRanges: Resetting '%s' axis ...", sparse->GetAxis(i)->GetName());
1298 sparse->GetAxis(i)->SetRange(0, 0);
1301 NLogTrace(
"NUtils::SetAxisRanges: Resetting '%s' axis [%d,%d] ...", sparse->GetAxis(i)->GetName(), 1,
1302 sparse->GetAxis(i)->GetNbins());
1303 sparse->GetAxis(i)->SetRange(1, sparse->GetAxis(i)->GetNbins());
1308 if (ranges.empty()) {
1309 NLogTrace(
"NUtils::SetAxisRanges: No axis ranges to set ...");
1312 TAxis * axis =
nullptr;
1313 TString title = sparse->GetTitle();
1314 for (
const auto & [key, val] : ranges) {
1315 NLogTrace(
"NUtils::SetAxisRanges: Setting axis range for axis %d to [%d,%d] ...", key, val[0], val[1]);
1316 axis = sparse->GetAxis(key);
1317 if (axis ==
nullptr) {
1318 NLogError(
"NUtils::SetAxisRanges: Axis %d is nullptr ...", key);
1321 NLogTrace(
"NUtils::SetAxisRanges: Setting axis range %s=[%d,%d] ...", axis->GetName(), val[0], val[1]);
1322 axis->SetRange(val[0], val[1]);
1323 if (axis->IsAlphanumeric()) {
1325 title += TString::Format(
" %s[%s]", axis->GetName(), axis->GetBinLabel(val[0]));
1328 title += TString::Format(
" %s[%0.2f - %0.2f]", axis->GetName(), axis->GetBinLowEdge(val[0]),
1329 axis->GetBinUpEdge(val[1]));
1333 if (modifyTitle) sparse->SetTitle(title.Data());
1334 NLogTrace(
"NUtils::SetAxisRanges: New title: %s", sparse->GetTitle());
1344 NLogError(
"Error: Axis is nullptr ...");
1350 NLogTrace(
"Getting axis range in base for '%s' rebin=%d rebin_start=%d bin=%d...", a->GetName(), rebin, rebin_start,
1353 min = rebin * (bin - 1) + rebin_start;
1354 max = min + rebin - 1;
1355 NLogTrace(
"Axis '%s' min=%d max=%d", a->GetName(), min, max);
1358 NLogError(
"Error: Axis '%s' min=%d is lower then 1 ...", a->GetName(), min);
1364 if (max > a->GetNbins()) {
1365 NLogError(
"Error: Axis '%s' max=%d is higher then %d ...", a->GetName(), max, a->GetNbins());
1379 int rebin = base->GetNbins() / a->GetNbins();
1382 int rebin_start = (base->GetNbins() % a->GetNbins()) + 1;
1383 rebin_start = rebin != 1 ? rebin_start : 1;
1385 NLogTrace(
"Getting axis range in base for '%s' min=%d max=%d rebin=%d rebin_start=%d...", a->GetName(), min, max,
1386 rebin, rebin_start);
1391 NLogTrace(
"Axis '%s' minBase=%d maxBase=%d", a->GetName(), minBase, maxBase);
1397 const std::string & fileName,
const std::vector<std::string> & axesNames)
1399 if (paths.empty()) {
1400 NLogError(
"Error: No paths provided ...");
1404 std::map<std::string, std::set<std::string>> axes;
1405 for (
const auto & path : paths) {
1406 NLogInfo(
"Found file: %s", path.c_str());
1408 TString relativePath = path;
1409 relativePath.ReplaceAll(findPath.c_str(),
"");
1410 relativePath.ReplaceAll(fileName.c_str(),
"");
1413 relativePath.ReplaceAll(
"//",
"/");
1415 relativePath.Remove(0, relativePath.BeginsWith(
"/") ? 1 : 0);
1417 relativePath.Remove(relativePath.EndsWith(
"/") ? relativePath.Length() - 1 : relativePath.Length(), 1);
1425 if (tokens.size() != axesNames.size()) {
1429 for (
size_t i = 0; i < tokens.size(); ++i) {
1430 axes[axesNames[i]].insert(tokens[i]);
1434 TObjArray * axesArr =
new TObjArray();
1435 for (
const auto & axisName : axesNames) {
1448 if (j.is_string()) {
1449 return j.get<std::string>();
1451 else if (j.is_number_integer()) {
1452 return std::to_string(j.get<
int>());
1454 else if (j.is_number_float()) {
1455 return std::to_string(j.get<
double>());
1457 else if (j.is_boolean()) {
1458 return j.get<
bool>() ?
"true" :
"false";
1460 else if (j.is_null()) {
1473 if (j.is_number_integer()) {
1474 return j.get<
int>();
1476 else if (j.is_number_float()) {
1477 return static_cast<int>(j.get<
double>());
1479 else if (j.is_boolean()) {
1480 return j.get<
bool>() ? 1 : 0;
1482 else if (j.is_null()) {
1496 if (j.is_number_float()) {
1497 return j.get<
double>();
1499 else if (j.is_number_integer()) {
1500 return static_cast<double>(j.get<
int>());
1502 else if (j.is_boolean()) {
1503 return j.get<
bool>() ? 1.0 : 0.0;
1505 else if (j.is_null()) {
1519 if (j.is_boolean()) {
1520 return j.get<
bool>();
1522 else if (j.is_number_integer()) {
1523 return j.get<
int>() != 0;
1525 else if (j.is_number_float()) {
1526 return j.get<
double>() != 0.0;
1528 else if (j.is_null()) {
1542 std::vector<std::string> out;
1544 for (
auto & v : j) {
1557 std::vector<int> v2;
1558 for (
int i = 0; i < size; i++) {
1559 v2.push_back(v1[i]);
1570 for (
size_t i = 0; i < v1.size(); i++) {
1579 std::stringstream msg;
1580 if (index >= 0) msg <<
"[" << std::setw(3) << std::setfill(
'0') << index <<
"] ";
1582 for (
size_t i = 0; i < coords.size(); ++i) {
1583 msg << std::setw(width) << std::setfill(
' ') << coords[i] << (i == coords.size() - 1 ?
"" :
",");
1593 std::stringstream msg;
1594 if (index >= 0) msg <<
"[" << std::setw(3) << std::setfill(
'0') << index <<
"] ";
1596 for (
size_t i = 0; i < coords.size(); ++i) {
1597 msg << std::setw(width) << std::setfill(
' ') << coords[i] << (i == coords.size() - 1 ?
"" :
",");
1607 std::stringstream msg;
1608 if (index >= 0) msg <<
"[" << std::setw(3) << std::setfill(
'0') << index <<
"] ";
1610 for (
size_t i = 0; i < coords.size(); ++i) {
1611 msg << std::setw(width) << std::setfill(
' ') << coords[i] << (i == coords.size() - 1 ?
"" :
",");
1621 std::stringstream msg;
1622 if (index >= 0) msg <<
"[" << std::setw(3) << std::setfill(
'0') << index <<
"] ";
1624 for (
size_t i = 0; i < coords.size(); ++i) {
1625 msg << std::setw(width) << std::setfill(
' ') << coords[i] << (i == coords.size() - 1 ?
"" :
",");
1644 std::vector<std::vector<int>> result;
1645 std::vector<int> current = v;
1646 std::sort(current.begin(), current.end());
1648 result.push_back(current);
1649 }
while (std::next_permutation(current.begin(), current.end()));
1653 for (
const auto & perm : result) {
1662 long long hours = seconds / 3600;
1664 long long minutes = seconds / 60;
1667 std::stringstream ss;
1668 ss << std::setw(2) << std::setfill(
'0') << hours <<
":" << std::setw(2) << std::setfill(
'0') << minutes <<
":"
1669 << std::setw(2) << std::setfill(
'0') << seconds;
1679 if (total == 0)
return;
1684 float percentage =
static_cast<float>(current) / total;
1685 int numChars =
static_cast<int>(percentage * barWidth);
1688 if (!prefix.empty()) std::cout <<
"[" << prefix <<
"]";
1691 for (
int i = 0; i < numChars; ++i) {
1694 for (
int i = 0; i < barWidth - numChars; ++i) {
1697 std::cout <<
"] " <<
static_cast<int>(percentage * 100.0) <<
"%"
1698 <<
" (" << current <<
"/" << total <<
")";
1699 if (!suffix.empty()) std::cout <<
" [" << suffix <<
"]";
1700 if (current == total) std::cout << std::endl;
1701 std::cout << std::flush;
1705 std::string prefix, std::string suffix,
int barWidth)
1710 if (total == 0)
return;
1712 if (current > total) current = total;
1714 float percentage =
static_cast<float>(current) / total;
1715 int numChars =
static_cast<int>(percentage * barWidth);
1719 if (!prefix.empty()) std::cout << prefix <<
"][";
1720 for (
int i = 0; i < numChars; ++i) {
1723 for (
int i = 0; i < barWidth - numChars; ++i) {
1726 std::cout <<
"] " << std::setw(3) <<
static_cast<int>(percentage * 100.0) <<
"%";
1729 auto currentTime = std::chrono::high_resolution_clock::now();
1730 auto elapsedSeconds = std::chrono::duration_cast<std::chrono::seconds>(currentTime - startTime).count();
1733 long long estimatedRemainingSeconds = 0;
1734 if (current > 0 && percentage > 0) {
1736 long long totalEstimatedSeconds =
static_cast<long long>(elapsedSeconds / percentage);
1737 estimatedRemainingSeconds = totalEstimatedSeconds - elapsedSeconds;
1740 std::cout <<
" (" << current <<
"/" << total <<
") "
1741 <<
"Elapsed: " <<
FormatTime(elapsedSeconds) <<
" "
1742 <<
"ETA: " <<
FormatTime(estimatedRemainingSeconds);
1743 if (!suffix.empty()) std::cout <<
" [" << suffix <<
"]";
1744 if (current == total) std::cout << std::endl;
1745 std::cout << std::flush;
1755 TCanvas * c =
new TCanvas(
"", title.c_str(), width, height);
1756 gROOT->GetListOfCanvases()->Remove(c);
1757 c->ResetBit(kMustCleanup);
1758 c->SetBit(kCanDelete, kFALSE);
1759 c->SetName(name.c_str());
1772 if (hns ==
nullptr) {
1773 NLogError(
"NUtils::CreateSparseFromParquetTaxi: THnSparse 'hns' is nullptr ...");
1777 std::shared_ptr<arrow::io::ReadableFile> infile;
1778 arrow::Result<std::shared_ptr<arrow::io::ReadableFile>> infile_result = arrow::io::ReadableFile::Open(filename);
1779 if (!infile_result.ok()) {
1780 NLogError(
"NUtils::CreateSparseFromParquetTaxi: Error opening file %s: %s", filename.c_str(),
1781 infile_result.status().ToString().c_str());
1784 infile = infile_result.ValueUnsafe();
1787 std::unique_ptr<parquet::arrow::FileReader> reader;
1790 arrow::Result<std::unique_ptr<parquet::arrow::FileReader>> reader_result =
1791 parquet::arrow::OpenFile(infile, arrow::default_memory_pool());
1792 if (!reader_result.ok()) {
1793 NLogError(
"NUtils::CreateSparseFromParquetTaxi: Error opening Parquet file reader for file %s: %s",
1794 filename.c_str(), reader_result.status().ToString().c_str());
1795 arrow::Status status = infile->Close();
1798 reader = std::move(reader_result).ValueUnsafe();
1802 std::shared_ptr<parquet::FileMetaData> file_metadata = reader->parquet_reader()->metadata();
1803 NLogTrace(
"Parquet file '%s' opened successfully.", filename.c_str());
1804 NLogTrace(
"Parquet file version: %d", file_metadata->version());
1805 NLogTrace(
"Parquet created by: %s", file_metadata->created_by().c_str());
1806 NLogTrace(
"Parquet number of columns: %d", file_metadata->num_columns());
1807 NLogTrace(
"Parquet number of rows: %lld", file_metadata->num_rows());
1808 NLogTrace(
"Parquet number of row groups: %d", file_metadata->num_row_groups());
1813 std::shared_ptr<arrow::RecordBatchReader> batch_reader;
1814 arrow::Status status = reader->GetRecordBatchReader(&batch_reader);
1816 NLogError(
"NUtils::CreateSparseFromParquetTaxi: Error reading table from Parquet file %s: %s", filename.c_str(),
1817 status.ToString().c_str());
1818 status = infile->Close();
1823 status = infile->Close();
1825 NLogWarning(
"NUtils::CreateSparseFromParquetTaxi: Error closing input file %s: %s", filename.c_str(),
1826 status.ToString().c_str());
1831 NLogTrace(
"Parquet Table Schema:\n%s", batch_reader->schema()->ToString().c_str());
1833 const Int_t nDims = hns->GetNdimensions();
1834 std::vector<std::string> column_names;
1835 for (
int i = 0; i < nDims; ++i) {
1836 column_names.push_back(hns->GetAxis(i)->GetName());
1842 max_rows = nMaxRows > 0 ? std::min(max_rows, nMaxRows) : max_rows;
1843 int print_rows = std::min(max_rows, 5);
1845 auto table_batch_reader = batch_reader;
1846 std::shared_ptr<arrow::RecordBatch> batch;
1847 auto point = std::make_unique<Double_t[]>(nDims);
1850 if (print_rows > 0) {
1851 NLogTrace(
"Printing first %d rows of Parquet file '%s' ...", print_rows, filename.c_str());
1855 int batch_count = 0;
1856 while (table_batch_reader->ReadNext(&batch).ok() && batch) {
1858 NLogTrace(
"Processing batch with %d rows and %d columns ...", batch->num_rows(), batch->num_columns());
1859 for (
int i = 0; i < batch->num_rows(); ++i) {
1860 if (i >= max_rows)
break;
1862 bool isValid =
true;
1864 for (
int j = 0; j < batch->num_columns(); ++j) {
1865 if (std::find(column_names.begin(), column_names.end(), batch->column_name(j)) == column_names.end())
1870 const auto & array = batch->column(j);
1871 arrow::Result<std::shared_ptr<arrow::Scalar>> scalar_result = array->GetScalar(i);
1872 if (scalar_result.ok()) {
1874 if (scalar_result.ValueUnsafe()->is_valid) {
1875 TAxis * axis = hns->GetAxis(idx);
1876 if (scalar_result.ValueUnsafe()->type->id() == arrow::Type::STRING ||
1877 scalar_result.ValueUnsafe()->type->id() == arrow::Type::LARGE_STRING) {
1880 std::string value = scalar_result.ValueUnsafe()->ToString();
1884 point[idx] = axis->GetBinCenter(axis->FindBin(value.c_str()));
1886 else if (scalar_result.ValueUnsafe()->type->id() == arrow::Type::INT32) {
1887 auto int_scalar = std::static_pointer_cast<arrow::Int32Scalar>(scalar_result.ValueUnsafe());
1889 point[idx] =
static_cast<Double_t
>(int_scalar->value);
1891 else if (scalar_result.ValueUnsafe()->type->id() == arrow::Type::INT64) {
1892 auto int64_scalar = std::static_pointer_cast<arrow::Int64Scalar>(scalar_result.ValueUnsafe());
1893 point[idx] =
static_cast<Double_t
>(int64_scalar->value);
1895 else if (scalar_result.ValueUnsafe()->type->id() == arrow::Type::UINT32) {
1896 auto uint32_scalar = std::static_pointer_cast<arrow::UInt32Scalar>(scalar_result.ValueUnsafe());
1897 point[idx] =
static_cast<Double_t
>(uint32_scalar->value);
1899 else if (scalar_result.ValueUnsafe()->type->id() == arrow::Type::FLOAT) {
1900 auto float_scalar = std::static_pointer_cast<arrow::FloatScalar>(scalar_result.ValueUnsafe());
1901 point[idx] =
static_cast<Double_t
>(float_scalar->value);
1903 else if (scalar_result.ValueUnsafe()->type->id() == arrow::Type::DOUBLE) {
1904 auto double_scalar = std::static_pointer_cast<arrow::DoubleScalar>(scalar_result.ValueUnsafe());
1905 point[idx] = double_scalar->value;
1908 NLogError(
"NUtils::CreateSparseFromParquetTaxi: Unsupported data type for column '%s' ...",
1909 batch->column_name(j).c_str());
1923 NLogError(
"NUtils::CreateSparseFromParquetTaxi: Error getting scalar at (%d,%d): %s", i, j,
1924 scalar_result.status().ToString().c_str());
1935 hns->Fill(point.get());
1938 NLogWarning(
"Skipping row %d due to invalid data.", i);
1948 NLogError(
"Parquet support is not enabled. Please compile with Parquet support.");
1955 if (objects.empty())
return;
1967 Bool_t prevMustClean = gROOT->MustClean();
1968 gROOT->SetMustClean(kFALSE);
1971 std::vector<TPad *> pads;
1972 for (
auto * obj : objects) {
1973 if (obj && obj->InheritsFrom(TPad::Class())) pads.push_back(
static_cast<TPad *
>(obj));
1975 for (
size_t i = 0; i < pads.size(); ++i) {
1976 TList * prims = pads[i]->GetListOfPrimitives();
1977 if (!prims || prims->IsEmpty())
continue;
1978 for (TObjLink * lnk = prims->FirstLink(); lnk; lnk = lnk->Next()) {
1979 TObject * child = lnk->GetObject();
1980 if (child && child->InheritsFrom(TPad::Class())) pads.push_back(
static_cast<TPad *
>(child));
1985 std::set<TObject *> inputSet(objects.begin(), objects.end());
1986 inputSet.erase(
nullptr);
1989 std::set<TObject *> orphans;
1990 for (
auto it = pads.rbegin(); it != pads.rend(); ++it) {
1991 TList * prims = (*it)->GetListOfPrimitives();
1992 if (!prims)
continue;
1994 for (TObjLink * lnk = prims->FirstLink(); lnk; lnk = lnk->Next()) {
1995 TObject * child = lnk->GetObject();
1996 if (child && inputSet.find(child) == inputSet.end()) orphans.insert(child);
1999 prims->UseRWLock(kFALSE);
2000 prims->SetOwner(kFALSE);
2001 prims->Clear(
"nodelete");
2005 for (
auto * obj : objects) {
2006 if (obj)
delete obj;
2011 for (
auto * obj : orphans) {
2015 gROOT->SetMustClean(prevMustClean);
2023 std::vector<TObject *> objects;
2024 for (TObjLink * lnk = lst->FirstLink(); lnk; lnk = lnk->Next()) {
2025 TObject * obj = lnk->GetObject();
2026 if (obj) objects.push_back(obj);
2030 lst->UseRWLock(kFALSE);
2031 lst->SetOwner(kFALSE);
2032 lst->Clear(
"nodelete");
2044 if (obj->InheritsFrom(TList::Class())) {
2045 TList * lst =
static_cast<TList *
>(obj);
2059 gSystem->GetProcInfo(&info);
2061 out[
"cpu_user"] = info.fCpuUser;
2062 out[
"cpu_sys"] = info.fCpuSys;
2063 out[
"cpu_total"] = info.fCpuUser + info.fCpuSys;
2064 out[
"mem_rss_kb"] = info.fMemResident;
2065 out[
"mem_vsize_kb"] = info.fMemVirtual;
2068 unsigned int hc = std::thread::hardware_concurrency();
2069 out[
"cpu_count"] = (hc == 0) ? 1 :
static_cast<int>(hc);
2077 out[
"totalRead"] = 0LL;
2078 out[
"totalWritten"] = 0LL;
2080 TList * files = (TList *)gROOT->GetListOfFiles();
2081 if (!files)
return out;
2083 Long64_t totalRead = 0;
2084 Long64_t totalWritten = 0;
2087 TObject * obj =
nullptr;
2088 while ((obj = next())) {
2089 TFile * f =
dynamic_cast<TFile *
>(obj);
2092 fi[
"name"] = f->GetName() ? f->GetName() :
"";
2093 fi[
"isZombie"] = (bool)f->IsZombie();
2094 fi[
"isOpen"] = (bool)f->IsOpen();
2097 Long64_t bytesRead = 0;
2098 Long64_t bytesWritten = 0;
2106 bytesRead = f->GetBytesRead();
2107 bytesWritten = f->GetBytesWritten();
2115 fi[
"bytesRead"] = bytesRead;
2116 fi[
"bytesWritten"] = bytesWritten;
2118 totalRead += bytesRead;
2119 totalWritten += bytesWritten;
2121 out[
"files"].push_back(fi);
2124 out[
"totalRead"] = totalRead;
2125 out[
"totalWritten"] = totalWritten;
2133 out[
"total_rx"] = 0ULL;
2134 out[
"total_tx"] = 0ULL;
2136 #if defined(__linux__)
2137 std::ifstream f(
"/proc/net/dev");
2138 if (!f.good())
return out;
2141 std::getline(f, line);
2142 std::getline(f, line);
2143 while (std::getline(f, line)) {
2144 if (line.empty())
continue;
2145 size_t colon = line.find(
':');
2146 if (colon == std::string::npos)
continue;
2147 std::string ifname = line.substr(0, colon);
2149 auto ltrim = [](std::string & s) {
2150 size_t start = s.find_first_not_of(
" \t");
2151 if (start != std::string::npos)
2152 s = s.substr(start);
2156 auto rtrim = [](std::string & s) {
2157 size_t end = s.find_last_not_of(
" \t");
2158 if (end != std::string::npos)
2159 s = s.substr(0, end + 1);
2165 std::string rest = line.substr(colon + 1);
2166 std::stringstream ss(rest);
2167 std::vector<unsigned long long> vals;
2171 vals.push_back(std::stoull(tok));
2174 vals.push_back(0ULL);
2177 if (vals.size() >= 9) {
2178 unsigned long long rx = vals[0];
2179 unsigned long long tx = vals[8];
2181 iface[
"name"] = ifname;
2184 out[
"interfaces"].push_back(iface);
2185 out[
"total_rx"] =
static_cast<unsigned long long>(
2186 out[
"total_rx"].is_null() ? 0ULL : out[
"total_rx"].get<
unsigned long long>()) +
2188 out[
"total_tx"] =
static_cast<unsigned long long>(
2189 out[
"total_tx"].is_null() ? 0ULL : out[
"total_tx"].get<
unsigned long long>()) +
2194 #elif defined(__APPLE__)
2195 struct ifaddrs * ifap =
nullptr;
2196 if (getifaddrs(&ifap) != 0)
return out;
2197 for (
struct ifaddrs * ifa = ifap; ifa; ifa = ifa->ifa_next) {
2198 if (!ifa->ifa_data)
continue;
2199 struct if_data * ifd = (
struct if_data *)ifa->ifa_data;
2201 unsigned long long rx = (
unsigned long long)ifd->ifi_ibytes;
2202 unsigned long long tx = (
unsigned long long)ifd->ifi_obytes;
2204 iface[
"name"] = ifa->ifa_name ? ifa->ifa_name : std::string();
2207 out[
"interfaces"].push_back(iface);
2209 static_cast<unsigned long long>(out[
"total_rx"].is_null() ? 0ULL : out[
"total_rx"].get<
unsigned long long>()) +
2212 static_cast<unsigned long long>(out[
"total_tx"].is_null() ? 0ULL : out[
"total_tx"].get<
unsigned long long>()) +
Provides HTTP request functionality using libcurl.
std::string get(const std::string &url, const std::string &cert_path="", const std::string &key_path="", const std::string &key_password_file="", bool insecure=false)
Performs an HTTP GET request.
int head(const std::string &url, const std::string &cert_path="", const std::string &key_path="", const std::string &key_password_file="", bool insecure=false)
Performs an HTTP HEAD request.
static std::mutex & GetLoggerMutex()
Get logger mutex reference.
Utility class providing static helper functions for file operations, histogram manipulations,...
static void GetTrueHistogramMinMax(const TH1 *h, double &min_val, double &max_val, bool include_overflow_underflow=false)
Get minimum and maximum value of histogram bins.
static bool SetAxisRanges(THnSparse *sparse, std::vector< std::vector< int >> ranges={}, bool withOverflow=false, bool modifyTitle=false, bool reset=true)
Set axis ranges for THnSparse using vector of ranges.
static TFile * OpenFile(std::string filename, std::string mode="READ", bool createLocalDir=true)
Open a ROOT file.
static void AddRawJsonInjection(json &j, const std::vector< std::string > &path, const std::string &rawJson, const std::string &injectionsKey="__raw_json_injections")
Add one raw JSON injection entry into metadata field.
static std::vector< std::string > Truncate(std::vector< std::string > values, std::string value)
Truncate vector of strings by a value.
static TH1 * ProjectTHnSparse(THnSparse *hns, const std::vector< int > &axes, Option_t *option="")
Project a THnSparse histogram onto specified axes.
static bool IsFileSupported(std::string filename)
Check if a file is supported.
static std::vector< std::string > FindEos(std::string path, std::string filename="")
Find EOS files in a path matching filename.
static bool LoadJsonFile(json &cfg, std::string filename)
Loads a JSON configuration file into the provided json object.
static json GetTFileIOStats()
Get TFile read/write statistics by inspecting ROOT's list of open files.
static bool SaveRawFile(std::string filename, std::string content)
Save content to a raw file.
static std::string OpenRawFile(std::string filename)
Open a raw file and return its content as string.
static std::vector< std::string > FindLocal(std::string path, std::string filename="")
Find local files in a path matching filename.
static void SafeDeleteTList(TList *&lst)
Safely delete a TList and all its contents, bypassing ROOT's GarbageCollect.
static void PrintPointSafe(const std::vector< int > &coords, int index=-1)
Print coordinates safely.
static TCanvas * CreateCanvas(const std::string &name, const std::string &title, int width=800, int height=600)
Create a ROOT TCanvas with specified name, title, and dimensions.
static std::string MergeRawJsonWithMetadata(const std::string &rawJson, const json &metadata)
Merge raw JSON string with metadata fields.
static THnSparse * ReshapeSparseAxes(THnSparse *hns, std::vector< int > order, std::vector< TAxis * > newAxes={}, std::vector< int > newPoint={}, Option_t *option="E")
Reshape axes of THnSparse.
static json GetNetDevStats()
Get system-wide network interface totals (RX/TX bytes) in a cross-platform way. On Linux reads /proc/...
static bool AccessPathName(std::string path)
Check if a path is accessible.
static std::vector< int > TokenizeInt(std::string_view input, const char delim)
Tokenize a string into integers by delimiter.
static THnSparse * Convert(TH1 *h1, std::vector< std::string > names={}, std::vector< std::string > titles={})
Convert TH1 to THnSparse.
static TMacro * OpenMacro(std::string filename)
Open a macro file.
static std::vector< std::string > Tokenize(std::string_view input, const char delim)
Tokenize a string by delimiter.
static bool CreateDirectory(const std::string &path)
static std::string FormatTime(long long seconds)
Format time in seconds to human-readable string.
static TAxis * CreateAxisFromLabels(const std::string &name, const std::string &title, const std::vector< std::string > &labels)
Create a TAxis from a list of labels.
static bool GetAxisRangeInBase(TAxis *a, int rebin, int rebin_start, int bin, int &min, int &max)
Get axis range in base for rebinned axis.
static json GetSystemStats()
Get process CPU and RSS memory statistics using ROOT's gSystem::GetProcInfo.
static std::set< std::string > Unique(std::vector< std::string > &paths, int axis, std::string path, char token='/')
Get unique values from vector of strings at specified axis.
static std::vector< std::string > GetJsonStringArray(json j)
Get JSON value as array of strings.
static std::string GetJsonString(json j)
Get JSON value as string.
static int Cp(std::string source, std::string destination, Bool_t progressbar=kTRUE)
Copy a file from source to destination.
static bool CollectRawJsonInjections(const json &j, RawJsonInjections &injections, const std::string &injectionsKey="__raw_json_injections")
Collect raw JSON injection entries from metadata field.
static bool EnableMT(Int_t numthreads=-1)
Enable multi-threading with specified number of threads.
static int GetJsonInt(json j)
Get JSON value as integer.
static std::string Join(const std::vector< std::string > &values, const char delim=',')
Join vector of strings into a single string with delimiter.
static void ProgressBar(int current, int total, std::string prefix="", std::string suffix="", int barWidth=50)
Display progress bar.
static std::string GetCoordsString(const std::vector< int > &coords, int index=-1, int width=0)
Get string representation of coordinates.
static void SafeDeleteObjects(std::vector< TObject * > &objects)
Safely delete a vector of ROOT objects, bypassing GarbageCollect.
static THnSparse * CreateSparseFromParquetTaxi(const std::string &filename, THnSparse *hns=nullptr, Int_t nMaxRows=-1)
Create THnSparse from Parquet Taxi file.
static void SafeDeleteObject(TObject *&obj)
Safely delete a TObject, handling TList contents and TCanvas/TPad cleanup.
static void VectorToArray(std::vector< int > v1, Int_t *v2)
Convert vector to array.
static std::vector< int > ArrayToVector(Int_t *v1, int size)
Convert array to vector.
static double GetJsonDouble(json j)
Get JSON value as double.
static bool GetJsonBool(json j)
Get JSON value as boolean.
static TObjArray * AxesFromDirectory(const std::vector< std::string > paths, const std::string &findPath, const std::string &fileName, const std::vector< std::string > &axesNames)
Creates an array of axes objects from files in specified directories.
static std::vector< std::string > Find(std::string path, std::string filename="")
Find files in a path matching filename.
static std::vector< std::vector< int > > Permutations(const std::vector< int > &v)
Generate all permutations of a vector.
static TAxis * CreateAxisFromLabelsSet(const std::string &name, const std::string &title, const std::set< std::string > &labels)
Create a TAxis from a set of labels.
static std::string InjectRawJson(json &j, const RawJsonInjections &injections)