11#include <TFileMerger.h>
19#include "RtypesCore.h"
35 TH1::AddDirectory(kFALSE);
48 std::string userConfigRaw, std::string binning,
bool show, std::string outfilename)
54 if (!
Core::LoadConfig(config, userConfig, environment, userConfigRaw, binning))
return 1;
56 if (!gCfg[
"ndmspc"][
"verbose"].is_null() && gCfg[
"ndmspc"][
"verbose"].is_number_integer())
57 fVerbose = gCfg[
"ndmspc"][
"verbose"].get<
int>();
59 if (!gCfg[
"ndmspc"][
"file"][
"cache"].is_null() && gCfg[
"ndmspc"][
"file"][
"cache"].is_string()) {
60 std::string cacheDir = gCfg[
"ndmspc"][
"file"][
"cache"].get<std::string>();
61 if (!cacheDir.empty()) {
62 Printf(
"Setting cache directory to '%s' ...", gSystem->ExpandPathName(cacheDir.c_str()));
63 TFile::SetCacheFileDir(gSystem->ExpandPathName(cacheDir.c_str()), 1, 1);
67 if (show) Printf(
"%s", gCfg.dump(2).c_str());
69 if (!outfilename.empty()) {
70 std::ofstream file(outfilename.c_str());
72 Printf(
"Config saved to file '%s' ...", outfilename.c_str());
84 if (
fVerbose >= 2) Printf(
"Ndmspc::PointRun::Init ...");
85 if (!gCfg[
"ndmspc"][
"process"][
"type"].get<std::string>().compare(
"all") &&
86 gCfg[
"ndmspc"][
"process"][
"ranges"].is_null() &&
87 !gCfg[
"ndmspc"][
"output"][
"delete"].get<std::string>().compare(
"onInit")) {
89 if (gCfg[
"ndmspc"][
"output"][
"host"].get<std::string>().empty()) {
90 gCfg[
"ndmspc"][
"output"][
"opt"] =
"";
93 std::string outFileName;
94 if (!gCfg[
"ndmspc"][
"output"][
"dir"].get<std::string>().empty()) {
97 outFileName += gCfg[
"ndmspc"][
"output"][
"dir"].get<std::string>();
100 std::string environment = gCfg[
"ndmspc"][
"environment"].get<std::string>();
101 outFileName += environment +
"/";
123 outFileName +=
"bins";
125 if (!extraPath.empty()) {
126 outFileName +=
"/" + extraPath;
127 outFileName.pop_back();
131 if (!outFileName.empty()) {
132 if (gCfg[
"ndmspc"][
"output"][
"host"].is_string() &&
133 !gCfg[
"ndmspc"][
"output"][
"host"].get<std::string>().empty()) {
135 Printf(
"Deleting output eos directory '%s' ...", outFileName.c_str());
137 TString::Format(
"%s/proc/user/?mgm.cmd=rm&mgm.path=%s&mgm.option=rf&mgm.format=json&filetype=raw",
138 gCfg[
"ndmspc"][
"output"][
"host"].get<std::string>().c_str(), outFileName.c_str())
141 if (
fVerbose >= 2) Printf(
"rmUrl '%s' ...", rmUrl.c_str());
144 Printf(
"Directory '%s' deleted", outFileName.c_str());
148 Printf(
"Directory '%s' deleted", outFileName.c_str());
149 gSystem->Exec(TString::Format(
"rm -rf %s", outFileName.c_str()));
154 if (
fVerbose >= 2) Printf(
"Ndmspc::PointRun::Init done ...");
165 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::OpenInputs");
169 if (gCfg[
"ndmspc"][
"data"][
"file"].get<std::string>().empty()) {
170 Printf(
"Error: Input file is empty !!! Aborting ...");
173 if (
fVerbose >= 0) Printf(
"Opening file '%s' ...", gCfg[
"ndmspc"][
"data"][
"file"].get<std::string>().c_str());
176 Printf(
"Error: Cannot open file '%s' !", gCfg[
"ndmspc"][
"data"][
"file"].get<std::string>().c_str());
183 for (
auto & obj : gCfg[
"ndmspc"][
"data"][
"objects"]) {
184 if (obj.get<std::string>().empty())
continue;
187 if (!gCfg[
"ndmspc"][
"data"][
"directory"].is_null() && gCfg[
"ndmspc"][
"data"][
"directory"].is_string())
188 dirName = gCfg[
"ndmspc"][
"data"][
"directory"].get<std::string>();
190 std::stringstream srcfull(obj.get<std::string>().c_str());
192 std::string srcName, sparseName;
194 getline(srcfull, srcName,
':');
195 getline(srcfull, sparseName,
':');
196 if (
fVerbose >= 2) Printf(
"srcName=%s customName=%s", srcName.c_str(), sparseName.c_str());
198 std::stringstream src(srcName.c_str());
202 while (getline(src, item,
'+')) {
205 if (!dirName.empty()) objName = dirName +
"/";
207 if (
fVerbose >= 1) Printf(
"Opening obj='%s' ...", objName.c_str());
210 s = (THnSparse *)
fInputFile->Get(objName.c_str());
212 if (
fVerbose >= 1) Printf(
"Warning: Cannot open object '%s' !!!", objName.c_str());
216 if (s && !sparseName.empty()) s->SetName(sparseName.c_str());
219 if (
fVerbose >= 1) Printf(
"Adding obj='%s' ...", objName.c_str());
220 stmp = (THnSparse *)
fInputFile->Get(objName.c_str());
221 if (stmp ==
nullptr) {
222 if (
fVerbose >= 1) Printf(
"Warning: Cannot open object '%s' !!!", objName.c_str());
233 Printf(
"Warning : Could not open '%s' from file '%s' !!! Skipping ...", obj.get<std::string>().c_str(),
234 gCfg[
"ndmspc"][
"data"][
"file"].get<std::string>().c_str());
239 TFunction * fun = gROOT->GetGlobalFunction(
"NdmspcOpenInputsUser",
nullptr, kTRUE);
241 gROOT->ProcessLine(TString::Format(
"NdmspcOpenInputsUser((Ndmspc::PointRun*)%p),",
this));
244 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::OpenInputs");
255 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::CreateResult fResultObject=%p", (
void *)
fResultObject);
259 THnSparse * s = (THnSparse *)
fInputList->At(0);
265 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
266 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
270 nDimsParams += gCfg[
"ndmspc"][
"result"][
"axes"].size();
272 const Int_t ndims = nDimsCuts + nDimsParams + nDimsProccess + 1;
274 Double_t xmin[ndims];
275 Double_t xmax[ndims];
276 std::vector<std::string> names;
277 std::vector<std::string> titles;
278 std::vector<TAxis *> cutAxes;
279 fMapAxesType =
new TH1S(
"mapAxesType",
"Type Axes Map", ndims, 0, ndims);
282 Int_t nParameters = gCfg[
"ndmspc"][
"result"][
"parameters"][
"labels"].size();
283 if (nParameters <= 0)
return nullptr;
287 bins[i] = nParameters;
289 xmax[i] = nParameters;
290 names.push_back(
"parameters");
291 titles.push_back(
"parameters");
300 bins[i] = a->GetNbins();
301 xmin[i] = a->GetXmin();
302 xmax[i] = a->GetXmax();
305 names.push_back(a->GetName());
306 std::string t = a->GetTitle();
307 if (t.empty()) t = a->GetName();
312 fMapAxesType->GetXaxis()->SetBinLabel(i + 1,
"sys-out");
317 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
318 if (
fVerbose >= 3) std::cout <<
"CreateResult() : " << cut.dump() << std::endl;
321 Int_t rebin_start = 1;
322 Int_t rebin_minimum = 1;
323 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
324 if (cut[
"rebin"].is_number_integer()) rebin = cut[
"rebin"].get<Int_t>();
325 if (cut[
"rebin_start"].is_number_integer()) rebin_start = cut[
"rebin_start"].get<Int_t>();
327 if (rebin_start > 1) {
328 rebin_minimum = (rebin_start % rebin);
331 TAxis * a = (TAxis *)s->GetListOfAxes()->FindObject(cut[
"axis"].get<std::string>().c_str());
332 if (a ==
nullptr)
return nullptr;
335 a = (TAxis *)a->Clone();
336 Int_t rebins_max = a->GetNbins() / rebin;
337 if ((a->GetNbins() - rebin_minimum + 1) % rebin == 0) rebins_max++;
338 Double_t arr[rebins_max];
344 for (i = rebin_minimum; i <= a->GetNbins(); i += rebin) {
346 arr[count++] = a->GetBinLowEdge(i);
349 if (count < rebins_max) arr[count++] = a->GetBinLowEdge(i);
353 a->Set(count - 1, arr);
356 bins[i] = a->GetNbins();
357 xmin[i] = a->GetXmin();
358 xmax[i] = a->GetXmax();
360 cutAxes.push_back(a);
361 names.push_back(a->GetName());
362 std::string t = a->GetTitle();
363 if (t.empty()) t = a->GetName();
370 for (
auto & value : gCfg[
"ndmspc"][
"result"][
"axes"]) {
371 if (!value[
"labels"].is_null()) {
372 bins[i] = value[
"labels"].size();
374 xmax[i] = value[
"labels"].size();
376 else if (!value[
"ranges"].is_null()) {
377 bins[i] = value[
"ranges"].size();
379 xmax[i] = value[
"ranges"].size();
382 Printf(
"Error: 'labels' or 'ranges' is missing !!!");
386 names.push_back(value[
"name"].get<std::string>().c_str());
387 titles.push_back(value[
"name"].get<std::string>().c_str());
394 THnSparse * fres =
new THnSparseD(
"results",
"Final results", i, bins, xmin, xmax);
396 Printf(
"Error: Could not create 'results' THnSparse object !!!");
400 TAxis * a = fres->GetAxis(iAxis);
402 Printf(
"Error: 'parameters' axis was not found !!!");
406 for (
auto & n : gCfg[
"ndmspc"][
"result"][
"parameters"][
"labels"]) {
407 a->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
408 a->SetBinLabel(iLablel++, n.get<std::string>().c_str());
418 if (axis->GetXbins()->GetArray()) fres->GetAxis(iAxis)->Set(axis->GetNbins(), axis->GetXbins()->GetArray());
419 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
423 if (!l.empty()) fres->GetAxis(iAxis)->SetBinLabel(iLabel, l.c_str());
431 for (
auto & a : cutAxes) {
433 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
434 if (a->GetXbins()->GetArray()) fres->GetAxis(iAxis)->Set(a->GetNbins(), a->GetXbins()->GetArray());
440 for (
auto & value : gCfg[
"ndmspc"][
"result"][
"axes"]) {
443 if (!value[
"labels"].is_null()) {
444 for (
auto & n : value[
"labels"]) {
445 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
446 fres->GetAxis(iAxis)->SetBinLabel(iLabel++, n.get<std::string>().c_str());
450 if (!value[
"ranges"].is_null()) {
451 for (
auto & n : value[
"ranges"]) {
452 fres->GetAxis(iAxis)->SetNameTitle(names.at(iAxis).c_str(), titles.at(iAxis).c_str());
453 fres->GetAxis(iAxis)->SetBinLabel(iLabel++, n[
"name"].get<std::string>().c_str());
459 if (
fVerbose >= 2) fres->Print(
"all");
463 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::CreateResult fResultObject=%p", (
void *)fres);
473 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::ApplyCuts");
476 TString titlePostfix =
"";
481 Int_t rebin_start = 1;
484 for (Long64_t i = 0; i <
fInputList->GetEntries(); i++) {
487 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
489 if (cut.is_null())
continue;
491 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
493 if (cut[
"rebin"].is_number_integer()) rebin = cut[
"rebin"].get<Int_t>();
494 if (cut[
"rebin_start"].is_number_integer()) rebin_start = cut[
"rebin_start"].get<Int_t>();
496 if (cut[
"axis"].is_string() && cut[
"axis"].get<std::string>().empty()) {
497 std::cerr <<
"Error: Axis name is empty ('" << cut <<
"') !!! Exiting ..." << std::endl;
500 if (cut[
"bin"][
"min"].get<Int_t>() < 0 || cut[
"bin"][
"max"].get<Int_t>() < 0) {
501 std::cerr <<
"Error: Bin min or max is less then 0 ('" << cut <<
"') !!! Exiting ..." << std::endl;
505 Int_t
id = s->GetListOfAxes()->IndexOf(s->GetListOfAxes()->FindObject(cut[
"axis"].get<std::string>().c_str()));
506 if (
id == s->GetListOfAxes()->GetEntries()) {
507 Printf(
"Axis '%s' was not found !!! Skipping ...", cut[
"axis"].get<std::string>().c_str());
514 Int_t binMin = cut[
"bin"][
"min"].get<Int_t>();
515 Int_t binMax = cut[
"bin"][
"max"].get<Int_t>();
521 Printf(
"cut=%s binLocal=%d binMin=%d binMax=%d", cut[
"axis"].get<std::string>().c_str(), binLocal, binMin,
523 s->GetAxis(
id)->SetRange(binMin, binMax);
526 if (s->GetAxis(
id)->IsAlphanumeric()) {
528 TString::Format(
"%s[%s bin=%d] ", s->GetAxis(
id)->GetName(), s->GetAxis(
id)->GetBinLabel(binMin), binMin);
531 titlePostfix += TString::Format(
"%s[%.2f,%.2f] ", s->GetAxis(
id)->GetName(),
532 s->GetAxis(
id)->GetBinLowEdge(binMin), s->GetAxis(
id)->GetBinUpEdge(binMax));
538 if (!titlePostfix.IsNull()) {
539 titlePostfix.Remove(titlePostfix.Length() - 1);
540 Printf(
"Processing '%s' ...", titlePostfix.Data());
541 gCfg[
"ndmspc"][
"projection"][
"title"] = titlePostfix.Data();
544 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::ApplyCuts");
553 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::ProcessSinglePoint");
568 json resultAxes = gCfg[
"ndmspc"][
"result"][
"axes"];
570 std::vector<std::string> names;
571 for (
auto & value : resultAxes) {
572 std::string n = value[
"name"].get<std::string>();
573 names.push_back(n.c_str());
578 if (
fVerbose >= 1) Printf(
"Starting User Process() ...");
582 if (
fVerbose >= 1) Printf(
"User Process() done ...");
588 if (
fVerbose >= 0) Printf(
"Skipping ...");
591 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::ProcessSinglePoint");
600 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::ProcessRecursive[%d]", i);
606 THnSparse * s = (THnSparse *)
fInputList->At(0);
607 TAxis * a = (TAxis *)s->GetListOfAxes()->FindObject(gCfg[
"ndmspc"][
"cuts"][i][
"axis"].get<std::string>().c_str());
609 Printf(
"Error: Axis canot be found");
613 Int_t end = a->GetNbins();
615 Int_t rebin_start = 1;
616 Int_t rebin_minimum = 1;
617 if (gCfg[
"ndmspc"][
"cuts"][i][
"rebin"].is_number_integer()) rebin = gCfg[
"ndmspc"][
"cuts"][i][
"rebin"].get<Int_t>();
618 if (gCfg[
"ndmspc"][
"cuts"][i][
"rebin_start"].is_number_integer())
619 rebin_start = gCfg[
"ndmspc"][
"cuts"][i][
"rebin_start"].get<Int_t>();
621 gCfg[
"ndmspc"][
"cuts"][i][
"rebin_minimum"] = rebin_minimum;
622 if (rebin > 1) end /= rebin;
623 if (rebin_start > 1) {
624 rebin_minimum = (rebin_start % rebin);
625 gCfg[
"ndmspc"][
"cuts"][i][
"rebin_minimum"] = rebin_minimum;
626 start = (rebin_start / rebin) + 1;
627 end = (a->GetNbins() - rebin_minimum + 1) / rebin;
633 if (gCfg[
"ndmspc"][
"process"][
"ranges"].is_array()) {
634 int range_min = gCfg[
"ndmspc"][
"process"][
"ranges"][i][0].get<
int>();
635 int range_max = gCfg[
"ndmspc"][
"process"][
"ranges"][i][1].get<
int>();
636 if (range_max > end || range_min < start || range_min > range_max || range_min > end || range_max < start) {
637 Printf(
"Error: Process range is out of bounds histogram(after rebin)=[%d,%d] request=[%d,%d] or requested min is "
638 "higher then requested max !!!",
639 start, end, range_min, range_max);
643 if (gCfg[
"ndmspc"][
"process"][
"ranges"][i][1] < end) end = range_max;
647 for (Int_t iBin = start; iBin <= end; iBin++) {
648 Int_t binMin = (iBin - 1) * rebin + rebin_minimum;
649 Int_t binMax = (iBin * rebin) - 1 + rebin_minimum;
651 Printf(
"axis=%s rebin=%d rebin_minimum=%d binMin=%d binMax=%d [%f,%f]", a->GetName(), rebin, rebin_minimum,
652 binMin, binMax, a->GetBinLowEdge(binMin), a->GetBinUpEdge(binMax));
653 gCfg[
"ndmspc"][
"cuts"][i][
"bin"][
"min"] = binMin;
654 gCfg[
"ndmspc"][
"cuts"][i][
"bin"][
"max"] = binMax;
658 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::ProcessRecursive[%d]", i);
668 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::ProcessRecursiveInner[%d]", i);
680 TList * outputList =
new TList();
686 if (
fVerbose >= 2) Printf(
"Running point macro '%s.C' ...",
fMacro->GetName());
693 Longptr_t ok = gROOT->ProcessLine(TString::Format(
"%s((Ndmspc::PointRun*)%p);",
fMacro->GetName(),
this));
705 if (ok &&
fVerbose >= 5) outputList->Print();
721 for (
int iPoint = 1; iPoint <
fResultObject->GetNdimensions(); iPoint++) {
742 std::string axisName = gCfg[
"ndmspc"][
"result"][
"axes"][i][
"name"].get<std::string>();
743 if (!gCfg[
"ndmspc"][
"result"][
"axes"][i][
"labels"].is_null()) {
744 for (
auto & v : gCfg[
"ndmspc"][
"result"][
"axes"][i][
"labels"]) {
745 TAxis * a = (TAxis *)
fResultObject->GetListOfAxes()->FindObject(axisName.c_str());
747 fCurrentPoint[id] = a->FindBin(v.get<std::string>().c_str());
753 else if (!gCfg[
"ndmspc"][
"result"][
"axes"][i][
"ranges"].is_null()) {
754 for (
auto & v : gCfg[
"ndmspc"][
"result"][
"axes"][i][
"ranges"]) {
755 TAxis * a = (TAxis *)
fResultObject->GetListOfAxes()->FindObject(axisName.c_str());
757 fCurrentPoint[id] = a->FindBin(v[
"name"].get<std::string>().c_str());
764 Printf(
"Error: ProcessRecursiveInner : No 'labels' or 'ranges' !!!");
769 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::ProcessRecursiveInner[%d]", i);
778 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::OutputFileOpen");
784 if (!gCfg[
"ndmspc"][
"output"][
"host"].get<std::string>().empty())
786 if (!gCfg[
"ndmspc"][
"output"][
"dir"].get<std::string>().empty())
820 if (cutsName.length() > 0) {
830 if (gCfg[
"ndmspc"][
"output"][
"post"].is_string()) {
831 std::string post = gCfg[
"ndmspc"][
"output"][
"post"].get<std::string>();
840 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
841 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
843 Int_t rebin_start = 1;
844 Int_t rebin_minimum = 1;
846 if (cut[
"rebin"].is_number_integer()) rebin = cut[
"rebin"].get<Int_t>();
847 if (cut[
"rebin_start"].is_number_integer()) rebin_start = cut[
"rebin_start"].get<Int_t>();
849 if (rebin_start > 1) {
850 rebin_minimum = (rebin_start % rebin);
851 if (rebin_minimum == 0) rebin_minimum = 1;
854 Int_t bin_min = cut[
"bin"][
"min"].get<Int_t>();
857 Int_t bin_min_converted = (bin_min - rebin_minimum) / rebin + 1;
870 if (!gCfg[
"ndmspc"][
"output"][
"file"].get<std::string>().empty())
876 gCfg[
"ndmspc"][
"output"][
"opt"].get<std::string>().c_str())
885 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::OutputFileOpen");
893 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::OutputFileClose");
910 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::OutputFileClose");
918 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::Finish");
930 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::Finish");
941 if (gCfg[
"ndmspc"][
"process"][
"type"].is_string()) type = gCfg[
"ndmspc"][
"process"][
"type"].get<std::string>();
944 Printf(
"Warning: [ndmspc][process][type] is missing or is empty in configuration !!! Setting it ot 'single' ...");
946 gCfg[
"ndmspc"][
"process"][
"type"] = type;
950 if (inputList ==
nullptr)
return 1;
952 if (!type.compare(
"single")) {
955 else if (!type.compare(
"all")) {
958 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
959 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
962 gCfg[
"ndmspc"][
"cuts"] = cuts;
967 Printf(
"Error: Value [process][type]='%s' is not supported !!! Exiting ...", type.c_str());
987 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::ProcessHistogramRun");
989 std::string fileNameHistogram = gCfg[
"ndmspc"][
"data"][
"histogram"][
"file"].get<std::string>();
990 std::string objName = gCfg[
"ndmspc"][
"data"][
"histogram"][
"obj"].get<std::string>();
993 if (!fProccessHistogram) {
994 Printf(
"Error: Proccess input histogram file '%s' could not opened !!!", fileNameHistogram.c_str());
999 Printf(
"Error: Proccess input histogram object '%s' could not opened !!!", objName.c_str());
1003 for (
int iAxis = 0; iAxis < axesArray->GetEntries(); iAxis++) {
1005 TAxis * aTmp = (TAxis *)axesArray->At(iAxis);
1010 if (gCfg[
"ndmspc"][
"data"][
"histogram"][
"bins"].is_array()) {
1012 for (
auto & v : gCfg[
"ndmspc"][
"data"][
"histogram"][
"bins"]) {
1013 Printf(
"%s", v.dump().c_str());
1016 for (
auto & idx : v) {
1033 for (
auto & p : proccessPoint) {
1036 path += std::to_string(std::abs(p)) +
"/";
1039 std::string fullPath = gCfg[
"ndmspc"][
"data"][
"histogram"][
"base"].get<std::string>();
1042 fullPath += gCfg[
"ndmspc"][
"data"][
"histogram"][
"filename"].get<std::string>();
1044 gCfg[
"ndmspc"][
"data"][
"file"] = fullPath;
1046 gCfg[
"ndmspc"][
"output"][
"post"] = path;
1057 Printf(
"Error: No entries in proccess histogram !!! Nothing to process !!!");
1061 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::ProcessHistogramRun");
1064bool PointRun::Run(std::string filename, std::string userConfig, std::string environment, std::string userConfigRaw,
1065 std::string binning,
bool show, std::string outfilename)
1072 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::Run");
1076 if (!
LoadConfig(filename, userConfig, environment, userConfigRaw, binning, show, outfilename))
return false;
1079 if (!gCfg[
"ndmspc"][
"data"][
"histogram"].is_null() && !gCfg[
"ndmspc"][
"data"][
"histogram"][
"enabled"].is_null() &&
1080 gCfg[
"ndmspc"][
"data"][
"histogram"][
"enabled"].get<bool>() ==
true) {
1086 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::Run");
1091 std::string userConfigRaw, std::string outfilename, std::string binnings)
1097 if (
fVerbose >= 2) Printf(
"[<-] Ndmspc::PointRun::GenerateJobs");
1101 if (!
LoadConfig(filename, userConfig, environment, userConfigRaw,
"",
true))
return false;
1103 if (outfilename[outfilename.size() - 1] ==
'/') outfilename.pop_back();
1105 Printf(
"Generating jobs with split '%s' with binnings '%s' to %s ...", jobs.c_str(), binnings.c_str(),
1106 outfilename.c_str());
1108 gCfg[
"ndmspc"][
"process"][
"type"] =
"all";
1110 std::vector<std::string> jobsArray =
Utils::Tokenize(jobs.c_str(),
':');
1111 std::vector<std::vector<int>> cutBins;
1113 std::vector<std::string> binningsArray =
Utils::Tokenize(binnings.c_str(),
',');
1115 std::string binningFirst =
"";
1116 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
1117 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
1118 if (!binningFirst.empty()) {
1119 binningFirst +=
"_";
1122 Int_t nbins = cut[
"nbins"].get<Int_t>();
1124 binningFirst += std::to_string(nbins) +
"-1";
1126 if (binningFirst.empty()) {
1127 Printf(
"Error: Binning is empty !!! Exiting ...");
1130 binningsArray.insert(binningsArray.begin(), binningFirst);
1132 for (
auto & binning : binningsArray) {
1133 std::vector<std::string> binningAxes =
Utils::Tokenize(binning.c_str(),
'_');
1138 for (
auto & cut : gCfg[
"ndmspc"][
"cuts"]) {
1140 if (cut[
"enabled"].is_boolean() && cut[
"enabled"].get<
bool>() ==
false)
continue;
1143 Int_t rebin_start = 1;
1144 Int_t rebin_minimum = 1;
1146 if (cut[
"rebin"].is_number_integer()) rebin = cut[
"rebin"].get<Int_t>();
1147 if (cut[
"rebin_start"].is_number_integer()) rebin_start = cut[
"rebin_start"].get<Int_t>();
1148 if (cut[
"nbins"].is_number_integer()) nbins = cut[
"nbins"].get<Int_t>();
1150 Printf(
"Error: Number of bins in '%s' is less then 0 !!! Exiting ...", cut[
"name"].get<std::string>().c_str());
1154 std::vector<std::string> binningAxis =
Utils::Tokenize(binningAxes[i],
'-');
1155 if (binningAxis.size() == 2) {
1156 rebin = atoi(binningAxis[0].c_str());
1157 rebin_start = atoi(binningAxis[1].c_str());
1158 gCfg[
"ndmspc"][
"cuts"][index][
"rebin"] = rebin;
1159 gCfg[
"ndmspc"][
"cuts"][index][
"rebin_start"] = rebin_start;
1163 if (rebin_start > 1) {
1164 rebin_minimum = (rebin_start % rebin);
1165 if (rebin_minimum == 0) rebin_minimum = 1;
1167 int start = (rebin_start / rebin) + 1;
1168 if (rebin == 1) start = rebin_start;
1169 int end = (nbins - rebin_minimum + 1) / rebin;
1171 Printf(
"Error: rebin=%d is higher then nbins=%d !!! Exiting ...", rebin, nbins);
1176 cutBins.push_back({start, end, atoi(jobsArray[i].c_str())});
1186 std::string outfilenameTmp = outfilename +
"/" + binning;
1188 std::this_thread::sleep_for(std::chrono::milliseconds(500));
1190 if (
fVerbose >= 2) Printf(
"[->] Ndmspc::PointRun::GenerateJobs");
1195 std::string & outfilename,
int & count)
1201 if (dim <
static_cast<Int_t
>(ranges.size())) {
1203 for (
int i = ranges[dim][0]; i <= ranges[dim][1]; i += ranges[dim][2]) {
1206 int max = i + ranges[dim][2] - 1;
1207 if (max > ranges[dim][1]) max = ranges[dim][1];
1208 cfg[
"ndmspc"][
"process"][
"ranges"][dim] = {min, max};
1214 std::string outFilenamePath = outfilename +
"/" + std::to_string(count++) +
".json";
1216 Printf(
"Jobs saved to '%s' '%s'", outFilenamePath.c_str(), gCfg[
"ndmspc"][
"process"][
"ranges"].dump().c_str());
1228 Printf(
"Genrating point run with name '%s' ...", name.c_str());
1239 "output": { "dir": "$HOME/.ndmspc/analyses/generated", "host": "" }
1242 "environment": "local",
1244 "file": "input.root",
1245 "objects": ["hNSparse"]
1249 "parameters": { "labels": ["Integral"], "default": "Integral" }
1254 "file": "content.root",
1261 "type": "error-only",
1262 "dir": "root://eos.ndmspc.io//eos/ndmspc/scratch/ndmspc/logs"
1271 std::string macroTemplateHeader = R""""(
1274#include <THnSparse.h>
1276#include <ndmspc/PointRun.h>
1277#include <ndmspc/Utils.h>
1280 std::string macroTemplate = R""""(
1282 json cfg = pr->Cfg();
1283 TList * inputList = pr->GetInputList();
1284 THnSparse * resultObject = pr->GetResultObject();
1285 Int_t * point = pr->GetCurrentPoint();
1286 std::vector<std::string> pointLabels = pr->GetCurrentPointLabels();
1287 json pointValue = pr->GetCurrentPointValue();
1288 TList * outputList = pr->GetOutputList();
1291 if (!cfg["user"]["verbose"].is_null() && cfg["user"]["verbose"].is_number_integer()) {
1292 verbose = cfg["user"]["verbose"].get<int>();
1295 THnSparse * hs = (THnSparse *)inputList->At(0);
1297 int projId = cfg["user"]["proj"].get<int>();
1298 TH1D * h = hs->Projection(projId, "O");
1300 TString titlePostfix = "(no cuts)";
1301 if (cfg["ndmspc"]["projection"]["title"].is_string())
1302 titlePostfix = cfg["ndmspc"]["projection"]["title"].get<std::string>();
1303 h->SetNameTitle("h", TString::Format("h - %s", titlePostfix.Data()).Data());
1306 // Skip bin (do not save any output)
1307 if (h->GetEntries() < cfg["user"]["minEntries"].get<int>())
1310 Double_t integral = h->Integral();
1312 Printf("Integral = %f ", integral);
1316 Ndmspc::Utils::SetResultValueError(cfg, resultObject, "Integral", point, integral, TMath::Sqrt(integral), false, true);
1319 if (!gROOT->IsBatch() && !cfg["ndmspc"]["process"]["type"].get<std::string>().compare("single")) {
1328 cfg["ndmspc"][
"data"][
"file"] = inFile.c_str();
1329 cfg[
"ndmspc"][
"data"][
"objects"] = {inObjectName.c_str()};
1331 if (cfg[
"ndmspc"][
"cuts"].size() == 0) {
1336 Printf(
"Error: Problem opening file '%s' !!! Exiting ...", inFile.c_str());
1340 THnSparse * tmpSparse = (THnSparse *)tmpFile->Get(inObjectName.c_str());
1342 Printf(
"Error: Problem opening object '%s' !!! Exiting ...", inObjectName.c_str());
1346 TObjArray * axes = tmpSparse->GetListOfAxes();
1348 for (
int i = 0; i < axes->GetEntries(); i++) {
1349 a = (TAxis *)axes->At(i);
1352 Printf(
"Init axis '%s' with enabled=false", a->GetName());
1353 cfg[
"ndmspc"][
"cuts"][i][
"enabled"] =
false;
1354 cfg[
"ndmspc"][
"cuts"][i][
"axis"] = a->GetName();
1355 cfg[
"ndmspc"][
"cuts"][i][
"bin"][
"min"] = 1;
1356 cfg[
"ndmspc"][
"cuts"][i][
"bin"][
"max"] = 1;
1357 cfg[
"ndmspc"][
"cuts"][i][
"rebin"] = 1;
1363 std::string outputConfig = name +
".json";
1364 std::ofstream fileConfig(outputConfig);
1365 fileConfig << std::setw(2) << cfg << std::endl;
1368 std::string outputMacro = name +
".C";
1369 std::ofstream fileMacro(outputMacro);
1370 fileMacro << macroTemplateHeader.c_str();
1371 fileMacro <<
"bool " << name.c_str() <<
"(Ndmspc::PointRun *pr)";
1372 fileMacro << macroTemplate.c_str();
1374 Printf(
"File '%s.C' and '%s.json' were generated ...", name.c_str(), name.c_str());
1378bool PointRun::Merge(
int from,
int to, std::string config, std::string userConfig, std::string environment,
1379 std::string userConfigRaw, std::string binning, std::string cacheDir, std::string fileOpt)
1385 if (from >= to && to >= 0) {
1386 Printf(
"Error: 'from=%d' must be smaller then 'to=%d' !!! Exiting ...", from, to);
1390 std::string fromFile =
"content.root";
1392 fromFile =
"merged_" + std::to_string(from) +
".root";
1394 std::string toFile =
"merged_" + std::to_string(to) +
".root";
1396 if (!
Core::LoadConfig(config, userConfig, environment, userConfigRaw, binning))
return false;
1398 if (!cacheDir.empty()) {
1399 cacheDir +=
"_" + std::to_string(gSystem->GetPid());
1400 Printf(
"Setting cache directory to '%s' ...", gSystem->ExpandPathName(cacheDir.c_str()));
1401 TFile::SetCacheFileDir(gSystem->ExpandPathName(cacheDir.c_str()), 1, 1);
1403 if (gCfg[
"ndmspc"][
"output"][
"host"].get<std::string>().empty()) {
1404 gCfg[
"ndmspc"][
"output"][
"opt"] =
"";
1407 std::string hostUrl = gCfg[
"ndmspc"][
"output"][
"host"].get<std::string>();
1413 if (!hostUrl.empty()) path = hostUrl +
"/";
1414 path += gCfg[
"ndmspc"][
"output"][
"dir"].get<std::string>() +
"/";
1416 path += environment +
"/";
1431 path = gSystem->ExpandPathName(path.c_str());
1434 std::vector<std::string> binsArrayTo;
1436 if (gCfg[
"ndmspc"][
"data"][
"histogram"][
"enabled"].get<bool>()) {
1437 std::string binToStr;
1438 for (
auto & bin : gCfg[
"ndmspc"][
"data"][
"histogram"][
"bins"]) {
1441 binsize = bin.size();
1442 if (to < 0) to = binsize + 1;
1446 int iLevelFrom = binsize - from + 1;
1447 int iLevelTo = binsize - to + 1;
1448 for (
auto & binElement : bin) {
1451 binToStr += std::to_string(binElement.get<Int_t>()) +
"/";
1453 if (iLevelFrom <= 0)
break;
1457 Printf(
"To: %s", binToStr.c_str());
1458 binsArrayTo.push_back(binToStr);
1459 if (binToStr.empty()) {
1460 toFile =
"results.root";
1466 binsArrayTo.push_back(
"");
1469 if (from > binsize) {
1470 Printf(
"Error: 'from=%d' must be smaller then 'binsize=%d' !!! Exiting ...", from, binsize);
1474 std::string pathBase = path;
1475 for (
auto & bin : binsArrayTo) {
1477 std::string pathFrom = pathBase +
"bins/" + bin;
1478 std::string outFile = pathBase;
1479 if (to <= binsize) {
1480 outFile +=
"bins/" + bin;
1484 TUrl url(pathFrom.c_str());
1485 std::string outHost = url.GetHost();
1486 std::string inputDirectory = url.GetFile();
1487 std::string linesMerge =
"";
1489 if (outHost.empty()) {
1490 if (gSystem->AccessPathName(pathFrom.c_str())) {
1491 Printf(
"Error: Nothing to merge, because path '%s' does not exist !!!", pathFrom.c_str());
1494 Printf(
"Doing local find %s -name %s", pathFrom.c_str(),
1495 gCfg[
"ndmspc"][
"output"][
"file"].get<std::string>().c_str());
1496 linesMerge = gSystem->GetFromPipe(TString::Format(
"find %s -name %s", pathFrom.c_str(),
1497 gCfg[
"ndmspc"][
"output"][
"file"].get<std::string>().c_str())
1504 Printf(
"Doing eos find -f %s", pathFrom.c_str());
1507 std::string contentFile = fromFile;
1511 std::vector<std::string> tokens;
1513 if (!inputDirectory.empty()) {
1515 TString::Format(
"root://%s//proc/user/"
1516 "?mgm.cmd=find&mgm.find.match=%s&mgm.path=%s&mgm.format=json&mgm.option=f&filetype=raw",
1517 outHost.c_str(), contentFile.c_str(), inputDirectory.c_str());
1525 int buffsize = 4096;
1526 char buff[buffsize + 1];
1528 Long64_t buffread = 0;
1529 std::string content;
1530 while (buffread < f->GetSize()) {
1532 if (buffread + buffsize > f->GetSize()) buffsize = f->GetSize() - buffread;
1535 f->ReadBuffer(buff, buffread, buffsize);
1536 buff[buffsize] =
'\0';
1538 buffread += buffsize;
1543 std::string ss =
"mgm.proc.stdout=";
1544 size_t pos = ss.size() + 1;
1545 content = content.substr(pos);
1548 std::stringstream check1(content);
1550 std::string intermediate;
1553 while (getline(check1, intermediate,
'&')) {
1554 tokens.push_back(intermediate);
1558 tokens.push_back(contentFile.c_str());
1560 linesMerge = tokens[0];
1563 if (linesMerge.empty()) {
1564 Printf(
"Error: Nothing to merge, because path '%s' does not contain file '%s' !!!", pathFrom.c_str(),
1569 std::stringstream check2(linesMerge);
1571 std::string tempMergeDir =
"/tmp";
1572 if (gCfg[
"ndmspc"][
"output"][
"merge"][
"tmpdir"].is_string())
1573 tempMergeDir = gCfg[
"ndmspc"][
"output"][
"merge"][
"tmpdir"].get<std::string>();
1575 std::string outFileLocal = tempMergeDir +
"/ndmspc-merged-" + std::to_string(gSystem->GetPid()) +
".root";
1577 if (hostUrl.empty()) {
1578 outFileLocal = outFile;
1581 outFileLocal = gSystem->ExpandPathName(outFileLocal.c_str());
1582 Printf(
"Output file: '%s'", outFile.c_str());
1584 TFileMerger m(kFALSE);
1585 m.OutputFile(TString::Format(
"%s%s", outFileLocal.c_str(), fileOpt.c_str()));
1590 while (std::getline(check2, line)) {
1592 if (!outHost.empty()) {
1593 line = TString::Format(
"root://%s/%s", outHost.c_str(), line.c_str()).Data();
1595 Printf(
"Adding file '%s' ...", line.data());
1596 m.AddFile(line.c_str());
1599 Printf(
"Merging ...");
1602 if (copy && toFile > 0) {
1603 Printf(
"Copy '%s' to '%s' ...", outFileLocal.c_str(), outFile.c_str());
1604 TFile::Cp(outFileLocal.c_str(), outFile.c_str());
1605 std::string rm =
"rm -f " + outFileLocal;
1606 Printf(
"Doing '%s' ...", rm.c_str());
1607 gSystem->Exec(rm.c_str());
1609 Printf(
"Output: '%s'", outFile.c_str());
1610 std::string rmCache =
"rm -rf ";
1611 rmCache += gSystem->ExpandPathName(cacheDir.c_str());
1612 Printf(
"Removing cache dir: '%s' ...", rmCache.c_str());
1613 gSystem->Exec(rmCache.c_str());
static bool SaveConfig(json cfg, std::string filename)
static bool LoadConfig(std::string config, std::string userConfig, std::string &environment, std::string userConfigRaw="", std::string binning="")
bool ProcessRecursiveInner(Int_t i, std::vector< std::string > &n)
std::string fCurrentOutputFileName
Current output filename.
bool GenerateRecursiveConfig(Int_t dim, std::vector< std::vector< int > > &ranges, json &cfg, std::string &outfilename, int &count)
TFile * fInputFile
Input file.
static std::string fgEnvironment
Environment.
THnSparse * fCurrentProccessHistogram
Current processed histogram.
bool Init(std::string extraPath="")
int fBinCount
Bin Count (TODO! rename to axis level maybe)
static bool Merge(int from=0, int to=1, std::string name="myAnalysis.json", std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string binning="", std::string cacheDir="${PWD}/.ndmspc_merge_cache", std::string fileOpt="?remote=1")
Merge.
int ProcessHistogramRun()
bool LoadConfig(std::string config, std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string binning="", bool show=false, std::string outfilename="")
THnSparse * CreateResult()
PointRun(std::string macro="NdmspcPointRun.C")
TH1S * fMapAxesType
Map axis type histogram.
bool Run(std::string filename, std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string binnings="", bool show=false, std::string outfilename="")
TList * fInputList
Input list.
std::vector< std::string > fCurrentPointLabels
Current labels.
THnSparse * fResultObject
Result object.
int fVerbose
Verbose level.
json fCurrentPointValue
Current point value.
void SetOutputList(TList *outList)
Sets output list.
bool fIsProcessOk
Flag that process is ok.
bool fIsSkipBin
Flag to skip bin.
std::vector< TAxis * > fCurrentProcessHistogramAxes
Current process histogram axes.
bool fIsProcessExit
Flag to exit process.
TFile * fCurrentOutputFile
Current output file.
static bool Generate(std::string name="myAnalysis", std::string inFile="myFile.root", std::string inObjectName="myNDHistogram")
TDirectory * fCurrentOutputRootDirectory
Current output root directory.
Int_t fCurrentPoint[32]
Current point.
bool ProcessSinglePoint()
bool GenerateJobs(std::string jobs, std::string filename, std::string userConfig="", std::string environment="", std::string userConfigRaw="", std::string jobDir="/tmp/ndmspc-jobs", std::string binnings="")
std::vector< int > fCurrentProcessHistogramPoint
Current process histoghram point.
bool ProcessRecursive(int i)
static std::string GetCutsPath(json cuts)
static TFile * OpenFile(std::string filename, std::string mode="READ", bool createLocalDir=true)
static Int_t GetBinFromBase(Int_t bin, Int_t rebin, Int_t rebin_start)
static TMacro * OpenMacro(std::string filename)
static std::vector< std::string > Tokenize(std::string_view input, const char delim)