From abb56f76358556ce04e53aabb4ec466162efc5b4 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Fri, 20 Mar 2026 08:39:54 +0100 Subject: [PATCH 1/3] DPL Analysis: use Input/OutputSpec metadata to detect special tables instead of origins --- .../Core/include/Framework/AnalysisHelpers.h | 37 ++++--- .../Core/include/Framework/AnalysisManagers.h | 10 +- .../include/Framework/DanglingEdgesContext.h | 9 ++ .../Core/include/Framework/DataSpecViews.h | 23 +++++ Framework/Core/src/AnalysisSupportHelpers.cxx | 7 +- Framework/Core/src/ArrowSupport.cxx | 86 +++++++++------- Framework/Core/src/WorkflowHelpers.cxx | 99 ++++++++++++++----- Framework/Core/src/runDataProcessing.cxx | 6 +- 8 files changed, 194 insertions(+), 83 deletions(-) diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h index a01d14b6632a9..9ac78c0933600 100644 --- a/Framework/Core/include/Framework/AnalysisHelpers.h +++ b/Framework/Core/include/Framework/AnalysisHelpers.h @@ -358,14 +358,14 @@ template constexpr auto tableRef2InputSpec() { std::vector metadata; - auto m = getInputMetadata>::metadata>(); - metadata.insert(metadata.end(), m.begin(), m.end()); - auto ccdbMetadata = getCCDBMetadata>::metadata>(); - metadata.insert(metadata.end(), ccdbMetadata.begin(), ccdbMetadata.end()); - auto p = getExpressionMetadata>::metadata>(); - metadata.insert(metadata.end(), p.begin(), p.end()); - auto idx = getIndexMetadata>::metadata>(); - metadata.insert(metadata.end(), idx.begin(), idx.end()); + auto sources = getInputMetadata>::metadata>(); + metadata.insert(metadata.end(), sources.begin(), sources.end()); + auto ccdbURLs = getCCDBMetadata>::metadata>(); + metadata.insert(metadata.end(), ccdbURLs.begin(), ccdbURLs.end()); + auto expressions = getExpressionMetadata>::metadata>(); + metadata.insert(metadata.end(), expressions.begin(), expressions.end()); + auto indices = getIndexMetadata>::metadata>(); + metadata.insert(metadata.end(), indices.begin(), indices.end()); if constexpr (!soa::with_ccdb_urls>::metadata>) { metadata.emplace_back(framework::ConfigParamSpec{"schema", framework::VariantType::String, framework::serializeSchema(o2::aod::MetadataTrait>::metadata::getSchema()), {"\"\""}}); } @@ -382,11 +382,22 @@ constexpr auto tableRef2InputSpec() template constexpr auto tableRef2OutputSpec() { + std::vector metadata; + using md = typename o2::aod::MetadataTrait>::metadata; + if constexpr (soa::with_ccdb_urls) { + metadata.emplace_back("ccdb:", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""}); + } else if constexpr (soa::with_expression_pack) { + metadata.emplace_back("projectors", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""}); + } else if constexpr (soa::with_index_pack) { + metadata.emplace_back("index-records", framework::VariantType::Bool, true, framework::ConfigParamSpec::HelpString{"\"\""}); + } return framework::OutputSpec{ framework::OutputLabel{o2::aod::label()}, o2::aod::origin(), o2::aod::description(o2::aod::signature()), - R.version}; + R.version, + framework::Lifetime::Timeframe, + metadata}; } template @@ -504,14 +515,14 @@ struct OutputForTable { using table_t = decltype(typeWithRef()); using metadata = aod::MetadataTrait>::metadata; - static OutputSpec const spec() + static constexpr auto spec() { - return OutputSpec{OutputLabel{aod::label()}, o2::aod::origin(), o2::aod::description(o2::aod::signature()), table_t::ref.version}; + return soa::tableRef2OutputSpec(); } - static OutputRef ref() + static constexpr auto ref() { - return OutputRef{aod::label(), table_t::ref.version}; + return soa::tableRef2OutputRef(); } }; diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h index 121ce7f4b4a77..a411bd6eefb56 100644 --- a/Framework/Core/include/Framework/AnalysisManagers.h +++ b/Framework/Core/include/Framework/AnalysisManagers.h @@ -183,20 +183,20 @@ bool newDataframeCondition(InputRecord& record, C& conditionGroup) /// Outputs handling template -bool appendOutput(std::vector&, T&, uint32_t) +constexpr bool appendOutput(std::vector&, T&, uint32_t) { return false; } template -bool appendOutput(std::vector& outputs, T&, uint32_t) +constexpr bool appendOutput(std::vector& outputs, T&, uint32_t) { - outputs.emplace_back(OutputForTable::spec()); + outputs.emplace_back(soa::tableRef2OutputSpec()); return true; } template -bool appendOutput(std::vector& outputs, T& producesGroup, uint32_t hash) +constexpr bool appendOutput(std::vector& outputs, T& producesGroup, uint32_t hash) { homogeneous_apply_refs([&outputs, hash](auto& produces) { return appendOutput(outputs, produces, hash); }, producesGroup); return true; @@ -261,7 +261,7 @@ bool prepareOutput(ProcessingContext&, T&) template bool prepareOutput(ProcessingContext& context, T& produces) { - produces.resetCursor(std::move(context.outputs().make(OutputForTable::ref()))); + produces.resetCursor(std::move(context.outputs().make(soa::tableRef2OutputRef()))); return true; } diff --git a/Framework/Core/include/Framework/DanglingEdgesContext.h b/Framework/Core/include/Framework/DanglingEdgesContext.h index 90a88974db038..c5f54297ee746 100644 --- a/Framework/Core/include/Framework/DanglingEdgesContext.h +++ b/Framework/Core/include/Framework/DanglingEdgesContext.h @@ -33,15 +33,24 @@ struct OutputObjectInfo { // been requested and for which we will need to inject // some source device. struct DanglingEdgesContext { + // generic AOD tables std::vector requestedAODs; std::vector providedAODs; + // extension tables std::vector requestedDYNs; std::vector providedDYNs; + // index tables std::vector requestedIDXs; + std::vector providedIDXs; + // ccdb tables std::vector providedTIMs; std::vector requestedTIMs; + // output objects std::vector providedOutputObjHist; + // inputs for the extension spawner std::vector spawnerInputs; + // inputs for the index builder + std::vector builderInputs; // These are the timestamped tables which are required to // inject the the CCDB objecs. diff --git a/Framework/Core/include/Framework/DataSpecViews.h b/Framework/Core/include/Framework/DataSpecViews.h index b38866d8aa6fd..63da68ab5d53e 100644 --- a/Framework/Core/include/Framework/DataSpecViews.h +++ b/Framework/Core/include/Framework/DataSpecViews.h @@ -14,8 +14,31 @@ #include "Framework/DataSpecUtils.h" #include +namespace o2::framework::checks +{ +static auto has_params_with_name(std::string&& name) +{ + return [name](ConfigParamSpec const& p) { return p.name.compare(name) == 0; }; +} + +static auto has_params_with_name_starting(std::string&& name) +{ + return [name](ConfigParamSpec const& p) { return p.name.starts_with(name); }; +} +} // namespace o2::framework::checks + namespace o2::framework::views { +static auto filter_with_params_by_name(std::string&& name) +{ + return std::views::filter([name = std::move(name)](auto const& spec) mutable { return std::ranges::any_of(spec.metadata, checks::has_params_with_name(std::move(name))); }); +} + +static auto filter_with_params_by_name_starting(std::string&& name) +{ + return std::views::filter([name = std::move(name)](auto const& spec) mutable { return std::ranges::any_of(spec.metadata, checks::has_params_with_name_starting(std::move(name))); }); +} + static auto partial_match_filter(auto what) { return std::views::filter([what](auto const& t) -> bool { return DataSpecUtils::partialMatch(t, what); }); diff --git a/Framework/Core/src/AnalysisSupportHelpers.cxx b/Framework/Core/src/AnalysisSupportHelpers.cxx index 7453751315626..21f312cd576e0 100644 --- a/Framework/Core/src/AnalysisSupportHelpers.cxx +++ b/Framework/Core/src/AnalysisSupportHelpers.cxx @@ -175,9 +175,14 @@ void AnalysisSupportHelpers::addMissingOutputsToBuilder(std::vector c // FIXME: until we have a single list of pairs additionalInputs | views::partial_match_filter(AODOrigins) | + std::ranges::views::filter([](InputSpec const& input) { + return std::ranges::none_of(input.metadata, [](ConfigParamSpec const& p) { return (p.name.compare("projectors") == 0) || (p.name.compare("index-records") == 0); }); + }) | sinks::update_input_list{requestedAODs}; // update requestedAODs additionalInputs | - views::partial_match_filter(header::DataOrigin{"DYN"}) | + std::ranges::views::filter([](InputSpec const& input) { + return std::ranges::any_of(input.metadata, [](ConfigParamSpec const& p) { return p.name.compare("projectors") == 0; }); + }) | sinks::update_input_list{requestedDYNs}; // update requestedDYNs } diff --git a/Framework/Core/src/ArrowSupport.cxx b/Framework/Core/src/ArrowSupport.cxx index b701ba5f8e01c..dd09303dd5d0d 100644 --- a/Framework/Core/src/ArrowSupport.cxx +++ b/Framework/Core/src/ArrowSupport.cxx @@ -50,7 +50,6 @@ O2_DECLARE_DYNAMIC_LOG(rate_limiting); namespace o2::framework { - class EndOfStreamContext; class ProcessingContext; @@ -578,45 +577,80 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec() } }, .adjustTopology = [](WorkflowSpecNode& node, ConfigContext const& ctx) { auto& workflow = node.specs; - auto spawner = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-spawner"); }); - auto analysisCCDB = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-ccdb"); }); - auto builder = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-index-builder"); }); - auto writer = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-writer"); }); auto& dec = ctx.services().get(); dec.requestedAODs.clear(); dec.requestedDYNs.clear(); - dec.providedDYNs.clear(); - dec.providedTIMs.clear(); - dec.requestedTIMs.clear(); auto inputSpecLessThan = [](InputSpec const& lhs, InputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); }; auto outputSpecLessThan = [](OutputSpec const& lhs, OutputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); }; + auto builder = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-index-builder"); }); if (builder != workflow.end()) { // collect currently requested IDXs dec.requestedIDXs.clear(); + dec.providedIDXs.clear(); for (auto& d : workflow | views::exclude_by_name(builder->name)) { d.inputs | - views::partial_match_filter(header::DataOrigin{"IDX"}) | + views::filter_with_params_by_name("index-records") | sinks::update_input_list{dec.requestedIDXs}; + d.outputs | + views::filter_with_params_by_name("index-records") | + sinks::update_output_list{dec.providedIDXs}; } + std::ranges::sort(dec.requestedIDXs, inputSpecLessThan); + std::ranges::sort(dec.providedIDXs, outputSpecLessThan); + dec.builderInputs.clear(); + dec.requestedIDXs | + views::filter_not_matching(dec.providedIDXs) | + sinks::append_to{dec.builderInputs}; // recreate inputs and outputs builder->inputs.clear(); builder->outputs.clear(); + AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.builderInputs, dec.requestedAODs, dec.requestedDYNs, *builder); + if (!builder->inputs.empty()) { + // load real AlgorithmSpec before deployment + builder->algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkOnDemandTablesSupport", "IndexTableBuilder", ctx); + } + } + + auto analysisCCDB = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-ccdb"); }); + if (analysisCCDB != workflow.end()) { + dec.requestedTIMs.clear(); + dec.providedTIMs.clear(); + for (auto& d : workflow | views::exclude_by_name(analysisCCDB->name)) { + d.inputs | + views::filter_with_params_by_name_starting("ccdb:") | + sinks::update_input_list{dec.requestedTIMs}; + d.outputs | + views::filter_with_params_by_name_starting("ccdb:") | + sinks::append_to{dec.providedTIMs}; + } + std::ranges::sort(dec.requestedTIMs, inputSpecLessThan); + std::ranges::sort(dec.providedTIMs, outputSpecLessThan); + // Use ranges::to> in C++23... + dec.analysisCCDBInputs.clear(); + dec.requestedTIMs | + views::filter_not_matching(dec.providedTIMs) | + sinks::append_to{dec.analysisCCDBInputs}; + // recreate inputs and outputs + analysisCCDB->outputs.clear(); + analysisCCDB->inputs.clear(); + AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.analysisCCDBInputs, dec.requestedAODs, dec.requestedDYNs, *analysisCCDB); // load real AlgorithmSpec before deployment - builder->algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkOnDemandTablesSupport", "IndexTableBuilder", ctx); - AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.requestedIDXs, dec.requestedAODs, dec.requestedDYNs, *builder); + analysisCCDB->algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkCCDBSupport", "AnalysisCCDBFetcherPlugin", ctx); } + auto spawner = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-spawner"); }); if (spawner != workflow.end()) { + dec.providedDYNs.clear(); // collect currently requested DYNs for (auto& d : workflow | views::exclude_by_name(spawner->name)) { d.inputs | - views::partial_match_filter(header::DataOrigin{"DYN"}) | + views::filter_with_params_by_name("projectors") | sinks::update_input_list{dec.requestedDYNs}; d.outputs | - views::partial_match_filter(header::DataOrigin{"DYN"}) | + views::filter_with_params_by_name("projectors") | sinks::append_to{dec.providedDYNs}; } std::ranges::sort(dec.requestedDYNs, inputSpecLessThan); @@ -628,32 +662,14 @@ o2::framework::ServiceSpec ArrowSupport::arrowBackendSpec() // recreate inputs and outputs spawner->outputs.clear(); spawner->inputs.clear(); - - // load real AlgorithmSpec before deployment - spawner->algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkOnDemandTablesSupport", "ExtendedTableSpawner", ctx); AnalysisSupportHelpers::addMissingOutputsToSpawner({}, dec.spawnerInputs, dec.requestedAODs, *spawner); - } - - if (analysisCCDB != workflow.end()) { - for (auto& d : workflow | views::exclude_by_name(analysisCCDB->name)) { - d.inputs | views::partial_match_filter(header::DataOrigin{"ATIM"}) | sinks::update_input_list{dec.requestedTIMs}; - d.outputs | views::partial_match_filter(header::DataOrigin{"ATIM"}) | sinks::append_to{dec.providedTIMs}; + if (!spawner->inputs.empty()) { + // load real AlgorithmSpec before deployment + spawner->algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkOnDemandTablesSupport", "ExtendedTableSpawner", ctx); } - std::ranges::sort(dec.requestedTIMs, inputSpecLessThan); - std::ranges::sort(dec.providedTIMs, outputSpecLessThan); - // Use ranges::to> in C++23... - dec.analysisCCDBInputs.clear(); - dec.requestedTIMs | views::filter_not_matching(dec.providedTIMs) | sinks::append_to{dec.analysisCCDBInputs}; - - // recreate inputs and outputs - analysisCCDB->outputs.clear(); - analysisCCDB->inputs.clear(); - // load real AlgorithmSpec before deployment - // FIXME how can I make the lookup depend on DYN tables as well?? - analysisCCDB->algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkCCDBSupport", "AnalysisCCDBFetcherPlugin", ctx); - AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.analysisCCDBInputs, dec.requestedAODs, dec.requestedDYNs, *analysisCCDB); } + auto writer = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { return spec.name.starts_with("internal-dpl-aod-writer"); }); if (writer != workflow.end()) { workflow.erase(writer); } diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx index 2ef3df9426fde..680149fafbeae 100644 --- a/Framework/Core/src/WorkflowHelpers.cxx +++ b/Framework/Core/src/WorkflowHelpers.cxx @@ -234,6 +234,8 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext ctx.services().registerService(ServiceRegistryHelpers::handleForService(new DanglingEdgesContext)); auto& dec = ctx.services().get(); + std::vector DYNs; + std::vector requestedCCDBs; std::vector providedCCDBs; @@ -279,6 +281,24 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext bool hasConditionOption = false; for (size_t ii = 0; ii < processor.inputs.size(); ++ii) { auto& input = processor.inputs[ii]; + bool hasProjectors = false; + bool hasIndexRecords = false; + bool hasCCDBURLs = false; + // all three options are exclusive + for (auto const& p : input.metadata) { + if (p.name.compare("projectors") == 0) { + hasProjectors = true; + break; + } + if (p.name.compare("index-records") == 0) { + hasIndexRecords = true; + break; + } + if (p.name.starts_with("ccdb:")) { + hasCCDBURLs = true; + break; + } + } switch (input.lifetime) { case Lifetime::Timer: { auto concrete = DataSpecUtils::asConcreteDataMatcher(input); @@ -318,29 +338,49 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext case Lifetime::Optional: break; } - if (DataSpecUtils::partialMatch(input, AODOrigins)) { - DataSpecUtils::updateInputList(dec.requestedAODs, InputSpec{input}); - } - if (DataSpecUtils::partialMatch(input, header::DataOrigin{"DYN"})) { + if (hasProjectors) { DataSpecUtils::updateInputList(dec.requestedDYNs, InputSpec{input}); - } - if (DataSpecUtils::partialMatch(input, header::DataOrigin{"IDX"})) { + } else if (hasIndexRecords) { DataSpecUtils::updateInputList(dec.requestedIDXs, InputSpec{input}); - } - if (DataSpecUtils::partialMatch(input, header::DataOrigin{"ATIM"})) { + } else if (hasCCDBURLs) { DataSpecUtils::updateInputList(dec.requestedTIMs, InputSpec{input}); + } else if (DataSpecUtils::partialMatch(input, AODOrigins)) { + DataSpecUtils::updateInputList(dec.requestedAODs, InputSpec{input}); } } std::ranges::stable_sort(timer.outputs, [](OutputSpec const& a, OutputSpec const& b) { return *DataSpecUtils::getOptionalSubSpec(a) < *DataSpecUtils::getOptionalSubSpec(b); }); for (auto& output : processor.outputs) { - if (DataSpecUtils::partialMatch(output, AODOrigins)) { - dec.providedAODs.emplace_back(output); - } else if (DataSpecUtils::partialMatch(output, header::DataOrigin{"DYN"})) { + bool hasProjectors = false; + bool hasIndexRecords = false; + bool hasCCDBURLs = false; + // all three options are exclusive + for (auto const& p : output.metadata) { + if (p.name.compare("projectors") == 0) { + hasProjectors = true; + break; + } + if (p.name.compare("index-records") == 0) { + hasIndexRecords = true; + break; + } + if (p.name.starts_with("ccdb:")) { + hasCCDBURLs = true; + break; + } + } + if (DataSpecUtils::partialMatch(output, header::DataOrigin{"DYN"})) { + DYNs.emplace_back(output); + } + if (hasProjectors) { dec.providedDYNs.emplace_back(output); - } else if (DataSpecUtils::partialMatch(output, header::DataOrigin{"ATIM"})) { + } else if (hasCCDBURLs) { dec.providedTIMs.emplace_back(output); + } else if (hasIndexRecords) { + dec.providedIDXs.emplace_back(output); + } else if (DataSpecUtils::partialMatch(output, AODOrigins)) { + dec.providedAODs.emplace_back(output); } else if (DataSpecUtils::partialMatch(output, header::DataOrigin{"ATSK"})) { dec.providedOutputObjHist.emplace_back(output); auto it = std::ranges::find_if(dec.outObjHistMap, [&](auto&& x) { return x.id == hash; }); @@ -350,6 +390,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext it->bindings.push_back(output.binding.value); } } + if (output.lifetime == Lifetime::Condition) { providedCCDBs.push_back(output); } @@ -358,10 +399,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext auto inputSpecLessThan = [](InputSpec const& lhs, InputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); }; auto outputSpecLessThan = [](OutputSpec const& lhs, OutputSpec const& rhs) { return DataSpecUtils::describe(lhs) < DataSpecUtils::describe(rhs); }; - std::ranges::sort(dec.requestedDYNs, inputSpecLessThan); - std::ranges::sort(dec.requestedTIMs, inputSpecLessThan); - std::ranges::sort(dec.providedDYNs, outputSpecLessThan); - std::ranges::sort(dec.providedTIMs, outputSpecLessThan); DataProcessorSpec indexBuilder{ "internal-dpl-aod-index-builder", @@ -369,14 +406,18 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext {}, AlgorithmSpec::dummyAlgorithm(), // real algorithm will be set in adjustTopology {}}; - AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.requestedIDXs, dec.requestedAODs, dec.requestedDYNs, indexBuilder); + std::ranges::sort(dec.requestedIDXs, inputSpecLessThan); + std::ranges::sort(dec.providedIDXs, outputSpecLessThan); + dec.requestedIDXs | views::filter_not_matching(dec.providedIDXs) | sinks::append_to{dec.builderInputs}; + AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.builderInputs, dec.requestedAODs, dec.requestedDYNs, indexBuilder); + std::ranges::sort(dec.requestedTIMs, inputSpecLessThan); + std::ranges::sort(dec.providedTIMs, outputSpecLessThan); dec.requestedTIMs | views::filter_not_matching(dec.providedTIMs) | sinks::append_to{dec.analysisCCDBInputs}; - DeploymentMode deploymentMode = DefaultsHelpers::deploymentMode(); - if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS) { - AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.analysisCCDBInputs, dec.requestedAODs, dec.requestedTIMs, analysisCCDBBackend); - } + AnalysisSupportHelpers::addMissingOutputsToBuilder(dec.analysisCCDBInputs, dec.requestedAODs, dec.requestedDYNs, analysisCCDBBackend); + std::ranges::sort(dec.requestedDYNs, inputSpecLessThan); + std::ranges::sort(dec.providedDYNs, outputSpecLessThan); dec.requestedDYNs | views::filter_not_matching(dec.providedDYNs) | sinks::append_to{dec.spawnerInputs}; DataProcessorSpec aodSpawner{ @@ -386,6 +427,9 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext AlgorithmSpec::dummyAlgorithm(), // real algorithm will be set in adjustTopology {}}; AnalysisSupportHelpers::addMissingOutputsToSpawner({}, dec.spawnerInputs, dec.requestedAODs, aodSpawner); + + std::ranges::sort(dec.requestedAODs, inputSpecLessThan); + std::ranges::sort(dec.providedAODs, outputSpecLessThan); AnalysisSupportHelpers::addMissingOutputsToReader(dec.providedAODs, dec.requestedAODs, aodReader); std::ranges::sort(requestedCCDBs, inputSpecLessThan); @@ -409,6 +453,14 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext extraSpecs.push_back(indexBuilder); } + // add the Analysys CCDB backend which reads CCDB objects using a provided table + DeploymentMode deploymentMode = DefaultsHelpers::deploymentMode(); + if (deploymentMode != DeploymentMode::OnlineDDS && deploymentMode != DeploymentMode::OnlineECS) { + if (analysisCCDBBackend.outputs.empty() == false) { + extraSpecs.push_back(analysisCCDBBackend); + } + } + // add the reader if (aodReader.outputs.empty() == false) { auto tfnsource = std::ranges::find_if(workflow, [](DataProcessorSpec const& spec) { @@ -515,11 +567,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext DataSpecUtils::updateOutputList(workflow[enumCandidate].outputs, OutputSpec{{"ccdb-diststf"}, dstf, Lifetime::Timeframe}); } - // add the Analysys CCDB backend which reads CCDB objects using a provided table - if (analysisCCDBBackend.outputs.empty() == false) { - extraSpecs.push_back(analysisCCDBBackend); - } - // add the timer if (timer.outputs.empty() == false) { extraSpecs.push_back(timer); diff --git a/Framework/Core/src/runDataProcessing.cxx b/Framework/Core/src/runDataProcessing.cxx index 815fce47544d0..a919cd863df57 100644 --- a/Framework/Core/src/runDataProcessing.cxx +++ b/Framework/Core/src/runDataProcessing.cxx @@ -1678,15 +1678,15 @@ int runStateMachine(DataProcessorSpecs const& workflow, continue; } // ignore devices with no metadata in inputs - auto hasMetadata = std::any_of(device.inputs.begin(), device.inputs.end(), [](InputSpec const& spec) { + auto hasMetadata = std::ranges::any_of(device.inputs, [](InputSpec const& spec) { return spec.metadata.empty() == false; }); if (!hasMetadata) { continue; } // ignore devices with no control options - auto hasControls = std::any_of(device.inputs.begin(), device.inputs.end(), [](InputSpec const& spec) { - return std::any_of(spec.metadata.begin(), spec.metadata.end(), [](ConfigParamSpec const& param) { + auto hasControls = std::ranges::any_of(device.inputs, [](InputSpec const& spec) { + return std::ranges::any_of(spec.metadata, [](ConfigParamSpec const& param) { return param.type == VariantType::Bool && param.name.find("control:") != std::string::npos; }); }); From 1688f9c29b5e5ef9cc3569fbbfd5abfdbb2ba440 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Fri, 20 Mar 2026 08:39:54 +0100 Subject: [PATCH 2/3] DPL Analysis: use Input/OutputSpec metadata to detect special tables instead of origins From a51a5eba755ed63e39631e7826cdf14fe47abeb0 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Fri, 27 Mar 2026 11:42:18 +0100 Subject: [PATCH 3/3] DPL Analysis: remove special origins and add From<> version for auto-generated tables --- .../AnalysisSupport/src/AODWriterHelpers.cxx | 5 +- Framework/Core/include/Framework/ASoA.h | 262 ++++++++++-------- .../include/Framework/AnalysisDataModel.h | 34 ++- .../Core/include/Framework/AnalysisHelpers.h | 67 ++++- .../Core/include/Framework/AnalysisManagers.h | 14 +- .../Framework/AnalysisSupportHelpers.h | 6 +- Framework/Core/src/AnalysisDataModel.cxx | 14 +- Framework/Core/src/AnalysisSupportHelpers.cxx | 2 +- Framework/Core/src/WorkflowHelpers.cxx | 27 +- Framework/Core/test/test_Concepts.cxx | 2 +- Framework/Core/test/test_IndexBuilder.cxx | 6 +- .../TestWorkflows/src/o2TestMultisource.cxx | 6 +- 12 files changed, 263 insertions(+), 182 deletions(-) diff --git a/Framework/AnalysisSupport/src/AODWriterHelpers.cxx b/Framework/AnalysisSupport/src/AODWriterHelpers.cxx index b76ffca13977e..19cf7a1ebff7b 100644 --- a/Framework/AnalysisSupport/src/AODWriterHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODWriterHelpers.cxx @@ -21,7 +21,6 @@ #include "Framework/TableConsumer.h" #include "Framework/DataOutputDirector.h" #include "Framework/TableTreeHelpers.h" -#include "Framework/Monitoring.h" #include "Framework/Signpost.h" #include @@ -32,8 +31,6 @@ #include #include #include -#include -#include O2_DECLARE_DYNAMIC_LOG(histogram_registry); @@ -157,7 +154,7 @@ AlgorithmSpec AODWriterHelpers::getOutputTTreeWriter(ConfigContext const& ctx) } // skip non-AOD refs - if (!DataSpecUtils::partialMatch(*ref.spec, writableAODOrigins)) { + if (!DataSpecUtils::partialMatch(*ref.spec, AODOrigins)) { continue; } startTime = DataRefUtils::getHeader(ref)->startTime; diff --git a/Framework/Core/include/Framework/ASoA.h b/Framework/Core/include/Framework/ASoA.h index 7a3307ae1a58c..7738cab6b303f 100644 --- a/Framework/Core/include/Framework/ASoA.h +++ b/Framework/Core/include/Framework/ASoA.h @@ -395,6 +395,20 @@ static constexpr auto sourceSpec() { return fmt::format("{}/{}/{}/{}", label(), origin_str(), description_str(signature()), R.version); } + +/// Replace origins in the TableRef array +template ar, o2::aod::is_origin_hash O> +consteval auto replaceOrigin() +{ + std::array res; + for (auto i = 0U; i < N; ++i) { + res[i].label_hash = ar[i].label_hash; + res[i].desc_hash = ar[i].desc_hash; + res[i].origin_hash = O::hash; + res[i].version = ar[i].version; + } + return res; +} } // namespace o2::aod namespace o2::soa @@ -1311,6 +1325,11 @@ concept with_sources = requires { T::sources.size(); }; +template +concept with_sources_generator = requires(T t) { + t.template generateSources>(); +}; + template concept with_ccdb_urls = requires { T::ccdb_urls.size(); @@ -1318,7 +1337,7 @@ concept with_ccdb_urls = requires { template concept with_base_table = requires { - typename aod::MetadataTrait>::metadata::base_table_t; + typename aod::MetadataTrait>::metadata::base_table_t; }; template @@ -1384,9 +1403,10 @@ static constexpr std::string getLabelFromType() return getLabelForTable::first_t>(); } template + requires(!soa::is_iterator) static constexpr std::string getLabelFromType() { - return getLabelForTable>::metadata::base_table_t>(); + return getLabelForTable>::metadata::base_table_t>(); } template @@ -1811,17 +1831,17 @@ consteval auto computeOriginals() } template - requires((sizeof...(Ts) > 0) && (!o2::soa::is_column || ...)) + requires((sizeof...(Ts) > 0) && (!(o2::soa::is_column && ...))) consteval auto computeOriginals() { return o2::soa::mergeOriginals(); } -template refs> -consteval auto commonOrigin() -{ - return (refs | std::ranges::views::filter([](TableRef const& r) { return (!(r.origin_hash == "DYN"_h || r.origin_hash == "IDX"_h)); })).front().origin_hash; -} +// template refs> +// consteval auto commonOrigin() +// { +// return (refs | std::ranges::views::filter([](TableRef const& r) { return (!(r.origin_hash == "DYN"_h || r.origin_hash == "IDX"_h)); })).front().origin_hash; +// } /// A Table class which observes an arrow::Table and provides /// It is templated on a set of Column / DynamicColumn types. @@ -1837,7 +1857,8 @@ class Table static constexpr const auto originalLabels = [] refs, size_t... Is>(std::index_sequence) { return std::array{o2::aod::label()...}; }.template operator()(std::make_index_sequence()); - static constexpr const uint32_t binding_origin = commonOrigin(); + static constexpr const uint32_t binding_origin = originals[0].origin_hash; // commonOrigin(); + static constexpr header::DataOrigin binding_origin_ = o2::aod::Hash::origin; template bindings> requires(ref.origin_hash == "CONC"_h) @@ -1850,10 +1871,10 @@ class Table requires(ref.origin_hash == "JOIN"_h) static consteval auto isIndexTargetOf() { - return std::ranges::find_if(self_t::originals, - [](TableRef const& r) { - return std::ranges::find(bindings, r) != bindings.end(); - }) != self_t::originals.end(); + return std::ranges::any_of(self_t::originals, + [](TableRef const& r) { + return std::ranges::any_of(bindings, [&r](TableRef const& b) { return b == r; }); + }); } template bindings> @@ -1866,7 +1887,7 @@ class Table template static consteval bool hasOriginal() { - return std::find_if(originals.begin(), originals.end(), [](TableRef const& o) { return o.desc_hash == r.desc_hash; }) != originals.end(); + return std::ranges::any_of(originals, [](TableRef const& o) { return o.desc_hash == r.desc_hash; }); } using columns_t = decltype(getColumns()); @@ -2389,9 +2410,9 @@ namespace o2::aod O2ORIGIN("AOD"); O2ORIGIN("AOD1"); O2ORIGIN("AOD2"); -O2ORIGIN("DYN"); -O2ORIGIN("IDX"); -O2ORIGIN("ATIM"); +// O2ORIGIN("DYN"); +// O2ORIGIN("IDX"); +// O2ORIGIN("ATIM"); O2ORIGIN("JOIN"); O2HASH("JOIN/0"); O2ORIGIN("CONC"); @@ -3269,85 +3290,113 @@ consteval auto getIndexTargets() #define DECLARE_SOA_TABLE_STAGED(_BaseName_, _Desc_, ...) \ DECLARE_SOA_TABLE_STAGED_VERSIONED(_BaseName_, _Desc_, 0, __VA_ARGS__); -#define DECLARE_SOA_EXTENDED_TABLE_FULL(_Name_, _Label_, _OriginalTable_, _Origin_, _Desc_, _Version_, ...) \ +#define DECLARE_SOA_EXTENDED_TABLE_NG(_Name_, _OriginalTable_, _Desc_, _Version_, ...) \ O2HASH(_Desc_ "/" #_Version_); \ + O2HASH(#_Name_ "Extension"); \ template \ - using _Name_##ExtensionFrom = soa::Table, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O>; \ - using _Name_##Extension = _Name_##ExtensionFrom>; \ - template > \ - struct _Name_##ExtensionMetadataFrom : TableMetadata, __VA_ARGS__> { \ + using _Name_##ExtensionFrom = soa::Table, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O>; \ + using _Name_##Extension = _Name_##ExtensionFrom>; \ + struct _Name_##ExtensionMetadata : TableMetadata, __VA_ARGS__> { \ using base_table_t = _OriginalTable_; \ - using extension_table_t = _Name_##ExtensionFrom; \ + template \ + using extension_table_t_from = _Name_##ExtensionFrom; \ + using extension_table_t = _Name_##Extension; \ using expression_pack_t = framework::pack<__VA_ARGS__>; \ - static constexpr auto sources = _OriginalTable_::originals; \ + static constexpr auto N = _OriginalTable_::originals.size(); \ + template > \ + static consteval auto generateSources() \ + { \ + return _OriginalTable_##From::originals; \ + } \ }; \ - using _Name_##ExtensionMetadata = _Name_##ExtensionMetadataFrom>; \ template <> \ struct MetadataTrait> { \ using metadata = _Name_##ExtensionMetadata; \ }; \ template \ - using _Name_##From = o2::soa::JoinFull, _OriginalTable_, _Name_##ExtensionFrom>; \ - using _Name_ = _Name_##From>; + using _Name_##From = o2::soa::Join<_OriginalTable_##From, _Name_##ExtensionFrom>; \ + using _Name_ = _Name_##From>; #define DECLARE_SOA_EXTENDED_TABLE(_Name_, _Table_, _Description_, _Version_, ...) \ - O2HASH(#_Name_ "Extension"); \ - DECLARE_SOA_EXTENDED_TABLE_FULL(_Name_, #_Name_ "Extension", _Table_, "DYN", _Description_, _Version_, __VA_ARGS__) + DECLARE_SOA_EXTENDED_TABLE_NG(_Name_, _Table_, _Description_, _Version_, __VA_ARGS__) #define DECLARE_SOA_EXTENDED_TABLE_USER(_Name_, _Table_, _Description_, ...) \ - O2HASH(#_Name_ "Extension"); \ - DECLARE_SOA_EXTENDED_TABLE_FULL(_Name_, #_Name_ "Extension", _Table_, "AOD", "EX" _Description_, 0, __VA_ARGS__) - -#define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_FULL(_Name_, _Label_, _OriginalTable_, _Origin_, _Desc_, _Version_, ...) \ - O2HASH(_Desc_ "/" #_Version_); \ - template \ - using _Name_##CfgExtensionFrom = soa::Table, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O>; \ - using _Name_##CfgExtension = _Name_##CfgExtensionFrom>; \ - template > \ - struct _Name_##CfgExtensionMetadataFrom : TableMetadata, __VA_ARGS__> { \ - using base_table_t = _OriginalTable_; \ - using extension_table_t = _Name_##CfgExtensionFrom; \ - using placeholders_pack_t = framework::pack<__VA_ARGS__>; \ - using configurable_t = std::true_type; \ - static constexpr auto sources = _OriginalTable_::originals; \ - }; \ - using _Name_##CfgExtensionMetadata = _Name_##CfgExtensionMetadataFrom>; \ - template <> \ - struct MetadataTrait> { \ - using metadata = _Name_##CfgExtensionMetadata; \ - }; \ - template \ - using _Name_##From = o2::soa::JoinFull, _OriginalTable_, _Name_##CfgExtensionFrom>; \ - using _Name_ = _Name_##From>; + DECLARE_SOA_EXTENDED_TABLE_NG(_Name_, _Table_, "EX" _Description_, 0, __VA_ARGS__) + +#define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_NG(_Name_, _OriginalTable_, _Desc_, _Version_, ...) \ + O2HASH(_Desc_ "/" #_Version_); \ + O2HASH(#_Name_ "CfgExtension"); \ + template \ + using _Name_##CfgExtensionFrom = soa::Table, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O>; \ + using _Name_##CfgExtension = _Name_##CfgExtensionFrom>; \ + struct _Name_##CfgExtensionMetadata : TableMetadata, __VA_ARGS__> { \ + using base_table_t = _OriginalTable_; \ + template \ + using extension_table_t_from = _Name_##CfgExtensionFrom; \ + using extension_table_t = _Name_##CfgExtension; \ + using placeholders_pack_t = framework::pack<__VA_ARGS__>; \ + using configurable_t = std::true_type; \ + static constexpr auto N = _OriginalTable_::originals.size(); \ + template > \ + static consteval auto generateSources() \ + { \ + return _OriginalTable_##From::originals; \ + } \ + }; \ + template <> \ + struct MetadataTrait> { \ + using metadata = _Name_##CfgExtensionMetadata; \ + }; \ + template \ + using _Name_##From = o2::soa::Join<_OriginalTable_##From, _Name_##CfgExtensionFrom>; \ + using _Name_ = _Name_##From>; + +#define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE(_Name_, _OriginalTable_, _Description_, ...) \ + DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_NG(_Name_, _OriginalTable_, "EX" _Description_, 0, __VA_ARGS__) + +#define DECLARE_SOA_INDEX_TABLE_NG(_Name_, _Key_, _Version_, _Desc_, _Exclusive_, ...) \ + O2HASH(#_Name_); \ + O2HASH(_Desc_ "/" #_Version_); \ + struct _Name_##Metadata : o2::aod::TableMetadata, soa::Index<>, __VA_ARGS__> { \ + static constexpr bool exclusive = _Exclusive_; \ + template \ + using KeyFrom = _Key_##From; \ + using Key = _Key_; \ + using index_pack_t = framework::pack<__VA_ARGS__>; \ + template > \ + static consteval auto generateSources() \ + { \ + return [](framework::pack) { \ + constexpr auto first = o2::soa::mergeOriginals(); \ + constexpr auto second = o2::aod::filterForKey(); \ + return o2::aod::replaceOrigin(); \ + }(framework::pack<__VA_ARGS__>{}); \ + } \ + static constexpr auto N = [](framework::pack) { \ + constexpr auto a = o2::soa::mergeOriginals(); \ + return o2::aod::filterForKey(); \ + }(framework::pack<__VA_ARGS__>{}) \ + .size(); \ + }; \ + template <> \ + struct MetadataTrait> { \ + using metadata = _Name_##Metadata; \ + }; \ + template \ + using _Name_##From = o2::soa::IndexTable, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O, _Key_##From, __VA_ARGS__>; \ + using _Name_ = _Name_##From>; -#define DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE(_Name_, _Table_, _Description_, ...) \ - O2HASH(#_Name_ "CfgExtension"); \ - DECLARE_SOA_CONFIGURABLE_EXTENDED_TABLE_FULL(_Name_, #_Name_ "CfgExtension", _Table_, "AOD", "EX" _Description_, 0, __VA_ARGS__) - -#define DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, _Origin_, _Version_, _Desc_, _Exclusive_, ...) \ - O2HASH(#_Name_); \ - O2HASH(_Desc_ "/" #_Version_); \ - template > \ - struct _Name_##MetadataFrom : o2::aod::TableMetadata, soa::Index<>, __VA_ARGS__> { \ - static constexpr bool exclusive = _Exclusive_; \ - using Key = _Key_; \ - using index_pack_t = framework::pack<__VA_ARGS__>; \ - static constexpr const auto sources = [](framework::pack) { \ - constexpr auto a = o2::soa::mergeOriginals(); \ - return o2::aod::filterForKey(); \ - }(framework::pack<__VA_ARGS__>{}); \ - static_assert(sources.size() - Key::originals.size() + 1 == framework::pack_size(index_pack_t{}), "One of the referred tables does not have index to Key"); \ - }; \ - using _Name_##Metadata = _Name_##MetadataFrom>; \ - \ - template > \ - using _Name_##From = o2::soa::IndexTable, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O, _Key_, __VA_ARGS__>; \ - using _Name_ = _Name_##From>; \ - \ - template <> \ - struct MetadataTrait> { \ - using metadata = _Name_##Metadata; \ - }; +#define DECLARE_SOA_INDEX_TABLE(_Name_, _Key_, _Description_, ...) \ + DECLARE_SOA_INDEX_TABLE_NG(_Name_, _Key_, 0, _Description_, false, __VA_ARGS__) + +#define DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(_Name_, _Key_, _Description_, ...) \ + DECLARE_SOA_INDEX_TABLE_NG(_Name_, _Key_, 0, _Description_, true, __VA_ARGS__) + +#define DECLARE_SOA_INDEX_TABLE_USER(_Name_, _Key_, _Description_, ...) \ + DECLARE_SOA_INDEX_TABLE_NG(_Name_, _Key_, 0, _Description_, false, __VA_ARGS__) + +#define DECLARE_SOA_INDEX_TABLE_EXCLUSIVE_USER(_Name_, _Key_, _Description_, ...) \ + DECLARE_SOA_INDEX_TABLE_NG(_Name_, _Key_, 0, _Description_, true, __VA_ARGS__) // Declare were each row is associated to a timestamp column of an _TimestampSource_ // table. @@ -3359,9 +3408,10 @@ consteval auto getIndexTargets() template \ using _Name_##TimestampFrom = soa::Table, o2::aod::Hash<_Desc_ "/" #_Version_ ""_h>, O>; \ using _Name_##Timestamp = _Name_##TimestampFrom>; \ - template > \ - struct _Name_##TimestampMetadataFrom : TableMetadata, __VA_ARGS__> { \ - using base_table_t = _TimestampSource_; \ + struct _Name_##TimestampMetadata : TableMetadata, __VA_ARGS__> { \ + template > \ + using base_table_t = _TimestampSource_##From; \ + template > \ using extension_table_t = _Name_##TimestampFrom; \ static constexpr const auto ccdb_urls = [](framework::pack) { \ return std::array{Cs::query...}; \ @@ -3369,49 +3419,37 @@ consteval auto getIndexTargets() static constexpr const auto ccdb_bindings = [](framework::pack) { \ return std::array{Cs::mLabel...}; \ }(framework::pack<__VA_ARGS__>{}); \ - static constexpr auto sources = _TimestampSource_::originals; \ + template > \ + static constexpr auto sources = _TimestampSource_##From::originals; \ static constexpr auto timestamp_column_label = _TimestampColumn_::mLabel; \ /*static constexpr auto timestampColumn = _TimestampColumn_;*/ \ }; \ - using _Name_##TimestampMetadata = _Name_##TimestampMetadataFrom>; \ template <> \ struct MetadataTrait> { \ using metadata = _Name_##TimestampMetadata; \ }; \ template \ - using _Name_##From = o2::soa::JoinFull, _TimestampSource_, _Name_##TimestampFrom>; \ + using _Name_##From = o2::soa::Join<_TimestampSource_, _Name_##TimestampFrom>; \ using _Name_ = _Name_##From>; #define DECLARE_SOA_TIMESTAMPED_TABLE(_Name_, _TimestampSource_, _TimestampColumn_, _Version_, _Desc_, ...) \ O2HASH(#_Name_ "Timestamped"); \ - DECLARE_SOA_TIMESTAMPED_TABLE_FULL(_Name_, #_Name_ "Timestamped", _TimestampSource_, _TimestampColumn_, "ATIM", _Version_, _Desc_, __VA_ARGS__) - -#define DECLARE_SOA_INDEX_TABLE(_Name_, _Key_, _Description_, ...) \ - DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, "IDX", 0, _Description_, false, __VA_ARGS__) - -#define DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(_Name_, _Key_, _Description_, ...) \ - DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, "IDX", 0, _Description_, true, __VA_ARGS__) - -#define DECLARE_SOA_INDEX_TABLE_USER(_Name_, _Key_, _Description_, ...) \ - DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, "AOD", 0, _Description_, false, __VA_ARGS__) - -#define DECLARE_SOA_INDEX_TABLE_EXCLUSIVE_USER(_Name_, _Key_, _Description_, ...) \ - DECLARE_SOA_INDEX_TABLE_FULL(_Name_, _Key_, "AOD", 0, _Description_, true, __VA_ARGS__) + DECLARE_SOA_TIMESTAMPED_TABLE_FULL(_Name_, #_Name_ "Timestamped", _TimestampSource_, _TimestampColumn_, "AOD", _Version_, _Desc_, __VA_ARGS__) namespace o2::soa { -template -struct JoinFull : Table, D, o2::aod::Hash<"JOIN"_h>, Ts...> { - using base = Table, D, o2::aod::Hash<"JOIN"_h>, Ts...>; +template +struct Join : Table, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Ts...> { + using base = Table, o2::aod::Hash<"JOIN/0"_h>, o2::aod::Hash<"JOIN"_h>, Ts...>; - JoinFull(std::shared_ptr&& table, uint64_t offset = 0) + Join(std::shared_ptr&& table, uint64_t offset = 0) : base{std::move(table), offset} { if (this->tableSize() != 0) { bindInternalIndicesTo(this); } } - JoinFull(std::vector>&& tables, uint64_t offset = 0) + Join(std::vector>&& tables, uint64_t offset = 0) : base{ArrowHelpers::joinTables(std::move(tables), std::span{base::originalLabels}), offset} { if (this->tableSize() != 0) { @@ -3421,6 +3459,7 @@ struct JoinFull : Table, D, o2::aod::Hash<"JOIN"_h>, Ts. using base::bindExternalIndices; using base::bindInternalIndicesTo; static constexpr const uint32_t binding_origin = base::binding_origin; + static constexpr const header::DataOrigin binding_origin_ = base::binding_origin_; template void bindExternalIndices(TA*... current) @@ -3433,7 +3472,7 @@ struct JoinFull : Table, D, o2::aod::Hash<"JOIN"_h>, Ts. ...); } - using self_t = JoinFull; + using self_t = Join; using table_t = base; static constexpr const auto originals = base::originals; static constexpr const auto originalLabels = base::originalLabels; @@ -3497,13 +3536,12 @@ struct JoinFull : Table, D, o2::aod::Hash<"JOIN"_h>, Ts. template static consteval bool contains() { - return std::find_if(originals.begin(), originals.end(), [](TableRef const& ref) { return ref.desc_hash == T::ref.desc_hash; }) != originals.end(); + return [](std::index_sequence) { + return (std::ranges::any_of(originals, [](TableRef const& ref) { return ref.desc_hash == T::originals[Is].desc_hash; }) && ...); + }(std::make_index_sequence()); } }; -template -using Join = JoinFull, Ts...>; - template constexpr auto join(Ts const&... t) { @@ -3511,7 +3549,7 @@ constexpr auto join(Ts const&... t) } template -concept is_join = framework::specialization_of_template; +concept is_join = framework::specialization_of_template; template constexpr bool is_soa_join_v = is_join; @@ -3562,6 +3600,7 @@ class FilteredBase : public T using table_t = typename T::table_t; using T::originals; static constexpr const uint32_t binding_origin = T::binding_origin; + static constexpr const header::DataOrigin binding_origin_ = T::binding_origin_; template void bindExternalIndices(TA*... current) { @@ -4177,6 +4216,7 @@ struct IndexTable : Table { using rest_t = framework::pack; static constexpr const uint32_t binding_origin = Key::binding_origin; + static constexpr const header::DataOrigin binding_origin_ = Key::binding_origin_; template void bindExternalIndices(TA*... current) diff --git a/Framework/Core/include/Framework/AnalysisDataModel.h b/Framework/Core/include/Framework/AnalysisDataModel.h index 9f48685820634..a19bf598c6bb9 100644 --- a/Framework/Core/include/Framework/AnalysisDataModel.h +++ b/Framework/Core/include/Framework/AnalysisDataModel.h @@ -51,6 +51,8 @@ DECLARE_SOA_TABLE(BCFlags, "AOD", "BCFLAG", //! flag for tagging UPCs, joinable bc::Flags); using BCs = BCs_001; // current version +template +using BCsFrom = BCs_001From; using BC = BCs::iterator; namespace timestamp @@ -66,7 +68,7 @@ using BCsWithTimestamps = soa::Join; namespace soa { -extern template struct JoinFull, aod::BCs, aod::Timestamps>; +extern template struct Join; } namespace aod { @@ -514,11 +516,11 @@ DECLARE_SOA_TABLE_FULL(StoredTracksIU, "Tracks_IU", "AOD", "TRACK_IU", //! On di track::Sign, o2::soa::Marker<2>); -DECLARE_SOA_EXTENDED_TABLE(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking - aod::track::Pt, - aod::track::P, - aod::track::Eta, - aod::track::Phi); +DECLARE_SOA_EXTENDED_TABLE_NG(TracksIU, StoredTracksIU, "EXTRACK_IU", 0, //! Track parameters at inner most update (e.g. ITS) as it comes from the tracking + aod::track::Pt, + aod::track::P, + aod::track::Eta, + aod::track::Phi); DECLARE_SOA_TABLE_FULL(StoredTracksCov, "TracksCov", "AOD", "TRACKCOV", //! On disk version of the TracksCov table at collision vertex track::SigmaY, track::SigmaZ, track::SigmaSnp, track::SigmaTgl, track::Sigma1Pt, @@ -680,9 +682,9 @@ using Run2TrackExtra = Run2TrackExtras::iterator; } // namespace aod namespace soa { -extern template struct soa::JoinFull, aod::Tracks, aod::TracksExtra>; -extern template struct soa::JoinFull, aod::Tracks, aod::TracksCov, aod::TracksExtra>; -extern template struct soa::JoinFull, aod::TracksExtension, aod::StoredTracks>; +extern template struct soa::Join; +extern template struct soa::Join; +extern template struct soa::Join; } // namespace soa namespace aod { @@ -926,6 +928,8 @@ using MFTTracks = MFTTracks_001; using StoredMFTTracks = StoredMFTTracks_001; using MFTTrack = MFTTracks::iterator; +template +using MFTTracksFrom = MFTTracks_001From; namespace fwdtrack // Index to MFTtrack column must be defined after table definition. { @@ -1005,7 +1009,7 @@ using MFTTrackCovFwd = MFTTracksCov::iterator; } // namespace aod namespace soa { -extern template struct JoinFull, aod::FwdTracks, aod::FwdTracksCov>; +extern template struct Join; } namespace aod { @@ -2026,6 +2030,8 @@ DECLARE_SOA_EXTENDED_TABLE(McParticles_001, StoredMcParticles_001, "EXMCPARTICLE using StoredMcParticles = StoredMcParticles_001; using McParticles = McParticles_001; using McParticle = McParticles::iterator; +template +using McParticlesFrom = McParticles_001From; } // namespace aod namespace soa { @@ -2191,11 +2197,11 @@ DECLARE_SOA_INDEX_COLUMN(FDD, fdd); //! // First entry: Collision #define INDEX_LIST_RUN2 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FV0CId, indices::FDDId DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run2MatchedExclusive, BCs, "MA_RN2_EX", INDEX_LIST_RUN2); //! -DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs, "MA_RN2_SP", INDEX_LIST_RUN2); //! +DECLARE_SOA_INDEX_TABLE(Run2MatchedSparse, BCs_001, "MA_RN2_SP", INDEX_LIST_RUN2); //! #define INDEX_LIST_RUN3 indices::CollisionId, indices::ZdcId, indices::BCId, indices::FT0Id, indices::FV0AId, indices::FDDId DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(Run3MatchedExclusive, BCs, "MA_RN3_EX", INDEX_LIST_RUN3); //! -DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs, "MA_RN3_SP", INDEX_LIST_RUN3); //! +DECLARE_SOA_INDEX_TABLE(Run3MatchedSparse, BCs_001, "MA_RN3_SP", INDEX_LIST_RUN3); //! // First entry: BC DECLARE_SOA_INDEX_TABLE_EXCLUSIVE(MatchedBCCollisionsExclusive, BCs, "MA_BCCOL_EX", //! @@ -2225,8 +2231,8 @@ DECLARE_EQUIVALENT_FOR_INDEX(aod::StoredTracksIU, aod::McTrackLabels); DECLARE_EQUIVALENT_FOR_INDEX(aod::Collisions, aod::McCollisionLabels); // Joins with collisions (only for sparse ones) // NOTE: index table needs to be always last argument -extern template struct JoinFull, aod::Collisions, aod::Run2MatchedSparse>; -extern template struct JoinFull, aod::Collisions, aod::Run3MatchedSparse>; +extern template struct Join; +extern template struct Join; } // namespace soa namespace aod { diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h index 9ac78c0933600..bfc5a02891dad 100644 --- a/Framework/Core/include/Framework/AnalysisHelpers.h +++ b/Framework/Core/include/Framework/AnalysisHelpers.h @@ -79,6 +79,7 @@ auto makeEmptyTable(const char* name) } template + requires(soa::not_void>::metadata>) auto makeEmptyTable() { auto schema = std::make_shared(soa::createFieldsFromColumns(typename aod::MetadataTrait>::metadata::persistent_columns_t{})); @@ -93,6 +94,7 @@ auto makeEmptyTable(const char* name, framework::pack p) } template + requires(soa::not_void::metadata>) auto makeEmptyTable(const char* name) { auto schema = std::make_shared(soa::createFieldsFromColumns(typename aod::MetadataTrait::metadata::persistent_columns_t{})); @@ -216,6 +218,26 @@ inline constexpr auto getSourceSchemas() }.template operator()(); } +template > +inline constexpr auto getSources() +{ + return [] refs>() { + return [](std::index_sequence) { + return std::vector{soa::tableRef2ConfigParamSpec()...}; + }(std::make_index_sequence()); + }.template operator()()>(); +} + +template > +inline constexpr auto getSourceSchemas() +{ + return [] refs>() { + return [](std::index_sequence) { + return std::vector{soa::tableRef2Schema()...}; + }(std::make_index_sequence()); + }.template operator()()>(); +} + template inline constexpr auto getCCDBUrls() { @@ -257,7 +279,7 @@ inline constexpr auto getIndexMapping() using indices = T::index_pack_t; using Key = T::Key; [&idx](std::index_sequence) mutable { - constexpr auto refs = T::sources; + constexpr auto refs = T::generateSources(); ([&idx]() mutable { constexpr auto pos = o2::aod::MetadataTrait>::metadata::template getIndexPosToKey(); if constexpr (pos == -1) { @@ -272,6 +294,26 @@ inline constexpr auto getIndexMapping() return idx; } +template > +constexpr auto getInputMetadata() -> std::vector +{ + std::vector inputMetadata; + + auto inputSources = getSources(); + std::sort(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; }); + auto last = std::unique(inputSources.begin(), inputSources.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; }); + inputSources.erase(last, inputSources.end()); + inputMetadata.insert(inputMetadata.end(), inputSources.begin(), inputSources.end()); + + auto inputSchemas = getSourceSchemas(); + std::sort(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name < b.name; }); + last = std::unique(inputSchemas.begin(), inputSchemas.end(), [](framework::ConfigParamSpec const& a, framework::ConfigParamSpec const& b) { return a.name == b.name; }); + inputSchemas.erase(last, inputSchemas.end()); + inputMetadata.insert(inputMetadata.end(), inputSchemas.begin(), inputSchemas.end()); + + return inputMetadata; +} + template constexpr auto getInputMetadata() -> std::vector { @@ -293,7 +335,7 @@ constexpr auto getInputMetadata() -> std::vector } template - requires(!soa::with_sources) + requires(!(soa::with_sources || soa::with_sources_generator)) constexpr auto getInputMetadata() -> std::vector { return {}; @@ -358,7 +400,12 @@ template constexpr auto tableRef2InputSpec() { std::vector metadata; - auto sources = getInputMetadata>::metadata>(); + std::vector sources; + if constexpr (soa::with_sources>::metadata>) { + sources = getInputMetadata>::metadata>(); + } else if constexpr (soa::with_sources_generator>::metadata>) { + sources = getInputMetadata>::metadata, o2::aod::Hash>(); + } metadata.insert(metadata.end(), sources.begin(), sources.end()); auto ccdbURLs = getCCDBMetadata>::metadata>(); metadata.insert(metadata.end(), ccdbURLs.begin(), ccdbURLs.end()); @@ -555,7 +602,7 @@ concept is_produces_group = std::derived_from; template struct TableTransform { using metadata = M; - constexpr static auto sources = M::sources; + constexpr static auto sources = M::template generateSources>(); template static auto base_spec() @@ -589,23 +636,23 @@ struct TableTransform { /// This helper struct allows you to declare extended tables which should be /// created by the task (as opposed to those pre-defined by data model) template -concept is_spawnable = soa::has_metadata>> && soa::has_extension>::metadata>; +concept is_spawnable = soa::has_metadata>> && soa::has_extension>::metadata>; template -concept is_dynamically_spawnable = soa::has_metadata>> && soa::has_configurable_extension>::metadata>; +concept is_dynamically_spawnable = soa::has_metadata>> && soa::has_configurable_extension>::metadata>; template constexpr auto transformBase() { - using metadata = typename aod::MetadataTrait>::metadata; - return TableTransform{}; + using metadata = typename aod::MetadataTrait>::metadata; + return TableTransform>::ref>{}; } template struct Spawns : decltype(transformBase()) { using spawnable_t = T; using metadata = decltype(transformBase())::metadata; - using extension_t = typename metadata::extension_table_t; + using extension_t = typename metadata::template extension_table_t_from>; using expression_pack_t = typename metadata::expression_pack_t; static constexpr size_t N = framework::pack_size(expression_pack_t{}); @@ -655,7 +702,7 @@ struct Defines : decltype(transformBase()) { static constexpr bool delayed = DELAYED; using spawnable_t = T; using metadata = decltype(transformBase())::metadata; - using extension_t = typename metadata::extension_table_t; + using extension_t = typename metadata::template extension_table_t_from>; using placeholders_pack_t = typename metadata::placeholders_pack_t; static constexpr size_t N = framework::pack_size(placeholders_pack_t{}); diff --git a/Framework/Core/include/Framework/AnalysisManagers.h b/Framework/Core/include/Framework/AnalysisManagers.h index a411bd6eefb56..1873f33937742 100644 --- a/Framework/Core/include/Framework/AnalysisManagers.h +++ b/Framework/Core/include/Framework/AnalysisManagers.h @@ -275,10 +275,10 @@ bool prepareOutput(ProcessingContext& context, T& producesGroup) template bool prepareOutput(ProcessingContext& context, T& spawns) { - using metadata = o2::aod::MetadataTrait>::metadata; - auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals(context), std::span{metadata::base_table_t::originalLabels}); + using metadata = o2::aod::MetadataTrait>::metadata; + auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals>()>(context), std::span{metadata::base_table_t::originalLabels}); if (originalTable->num_rows() == 0) { - originalTable = makeEmptyTable(); + originalTable = makeEmptyTable("EMPTY", typename metadata::base_table_t::persistent_columns_t{}); } using D = o2::aod::Hash; @@ -295,17 +295,17 @@ template bool prepareOutput(ProcessingContext& context, T& builds) { using metadata = o2::aod::MetadataTrait>::metadata; - return builds.build(extractOriginals(context)); + return builds.build(extractOriginals>()>(context)); } template bool prepareOutput(ProcessingContext& context, T& defines) requires(T::delayed == false) { - using metadata = o2::aod::MetadataTrait>::metadata; - auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals(context), std::span{metadata::base_table_t::originalLabels}); + using metadata = o2::aod::MetadataTrait>::metadata; + auto originalTable = soa::ArrowHelpers::joinTables(extractOriginals>()>(context), std::span{metadata::base_table_t::originalLabels}); if (originalTable->num_rows() == 0) { - originalTable = makeEmptyTable(); + originalTable = makeEmptyTable("EMPTY", typename metadata::base_table_t::persistent_columns_t{}); } if (defines.inputSchema == nullptr) { defines.inputSchema = originalTable->schema(); diff --git a/Framework/Core/include/Framework/AnalysisSupportHelpers.h b/Framework/Core/include/Framework/AnalysisSupportHelpers.h index ef1d056b62f2b..803d8cf9d4685 100644 --- a/Framework/Core/include/Framework/AnalysisSupportHelpers.h +++ b/Framework/Core/include/Framework/AnalysisSupportHelpers.h @@ -21,11 +21,11 @@ namespace o2::framework { static constexpr std::array AODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}}; -static constexpr std::array extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}, header::DataOrigin{"AMD"}, header::DataOrigin{"EMB"}}; -static constexpr std::array writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"DYN"}}; +// static constexpr std::array extendedAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}}; +// static constexpr std::array writableAODOrigins{header::DataOrigin{"AOD"}, header::DataOrigin{"AOD1"}, header::DataOrigin{"AOD2"}, header::DataOrigin{"EMB"}}; class DataOutputDirector; -class ConfigContext; +struct ConfigContext; // Helper class to be moved in the AnalysisSupport plugin at some point struct AnalysisSupportHelpers { diff --git a/Framework/Core/src/AnalysisDataModel.cxx b/Framework/Core/src/AnalysisDataModel.cxx index 906772234d608..fbbabc35f0aab 100644 --- a/Framework/Core/src/AnalysisDataModel.cxx +++ b/Framework/Core/src/AnalysisDataModel.cxx @@ -12,12 +12,12 @@ namespace o2::soa { -template struct JoinFull, aod::BCs, aod::Timestamps>; -template struct JoinFull, aod::Tracks, aod::TracksExtra>; -template struct JoinFull, aod::Tracks, aod::TracksCov, aod::TracksExtra>; -template struct JoinFull, aod::FwdTracks, aod::FwdTracksCov>; -template struct JoinFull, aod::Collisions, aod::Run2MatchedSparse>; -template struct JoinFull, aod::Collisions, aod::Run3MatchedSparse>; +template struct Join; +template struct Join; +template struct Join; +template struct Join; +template struct Join; +template struct Join; -template struct JoinFull, aod::TracksExtension, aod::StoredTracks>; +template struct Join; } // namespace o2::soa diff --git a/Framework/Core/src/AnalysisSupportHelpers.cxx b/Framework/Core/src/AnalysisSupportHelpers.cxx index 21f312cd576e0..9e0f0ac3b778a 100644 --- a/Framework/Core/src/AnalysisSupportHelpers.cxx +++ b/Framework/Core/src/AnalysisSupportHelpers.cxx @@ -102,7 +102,7 @@ std::shared_ptr AnalysisSupportHelpers::getDataOutputDirecto // use the dangling outputs std::vector danglingOutputs; for (auto ii = 0u; ii < OutputsInputs.size(); ii++) { - if (DataSpecUtils::partialMatch(OutputsInputs[ii], writableAODOrigins) && isDangling[ii]) { + if (DataSpecUtils::partialMatch(OutputsInputs[ii], AODOrigins) && isDangling[ii]) { danglingOutputs.emplace_back(OutputsInputs[ii]); } } diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx index 680149fafbeae..a01916c9b3217 100644 --- a/Framework/Core/src/WorkflowHelpers.cxx +++ b/Framework/Core/src/WorkflowHelpers.cxx @@ -234,8 +234,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext ctx.services().registerService(ServiceRegistryHelpers::handleForService(new DanglingEdgesContext)); auto& dec = ctx.services().get(); - std::vector DYNs; - std::vector requestedCCDBs; std::vector providedCCDBs; @@ -370,9 +368,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext break; } } - if (DataSpecUtils::partialMatch(output, header::DataOrigin{"DYN"})) { - DYNs.emplace_back(output); - } if (hasProjectors) { dec.providedDYNs.emplace_back(output); } else if (hasCCDBURLs) { @@ -596,7 +591,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext continue; } // AODs are skipped in any case. - if (DataSpecUtils::partialMatch(dec.outputsInputs[ii], extendedAODOrigins)) { + if (DataSpecUtils::partialMatch(dec.outputsInputs[ii], AODOrigins)) { continue; } redirectedOutputsInputs.emplace_back(dec.outputsInputs[ii]); @@ -731,7 +726,7 @@ void WorkflowHelpers::injectAODWriter(WorkflowSpec& workflow, ConfigContext cons // select outputs of type AOD which need to be saved dec.outputsInputsAOD.clear(); for (auto ii = 0u; ii < dec.outputsInputs.size(); ii++) { - if (DataSpecUtils::partialMatch(dec.outputsInputs[ii], extendedAODOrigins)) { + if (DataSpecUtils::partialMatch(dec.outputsInputs[ii], AODOrigins)) { auto ds = dod->getDataOutputDescriptors(dec.outputsInputs[ii]); if (ds.size() > 0 || dec.isDangling[ii]) { dec.outputsInputsAOD.emplace_back(dec.outputsInputs[ii]); @@ -864,10 +859,8 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow, if (forwards.empty()) { errorDueToMissingOutputFor(consumer, input); } - availableOutputsInfo.erase(std::remove_if(availableOutputsInfo.begin(), availableOutputsInfo.end(), [](auto& info) { return info.enabled == false; }), availableOutputsInfo.end()); - for (auto& forward : forwards) { - availableOutputsInfo.push_back(forward); - } + availableOutputsInfo.erase(std::remove_if(availableOutputsInfo.begin(), availableOutputsInfo.end(), [](auto const& info) { return info.enabled == false; }), availableOutputsInfo.end()); + std::ranges::copy(forwards, std::back_inserter(availableOutputsInfo)); } O2_SIGNPOST_END(workflow_helpers, sid, "input matching", ""); } @@ -946,14 +939,14 @@ WorkflowParsingState WorkflowHelpers::verifyWorkflow(const o2::framework::Workfl return WorkflowParsingState::Empty; } std::set validNames; - std::vector availableOutputs; - std::vector requiredInputs; + // std::vector availableOutputs; + // std::vector requiredInputs; // An index many to one index to go from a given input to the // associated spec - std::map inputToSpec; + // std::map inputToSpec; // A one to one index to go from a given output to the Spec emitting it - std::map outputToSpec; + // std::map outputToSpec; std::ostringstream ss; @@ -1134,14 +1127,14 @@ void WorkflowHelpers::validateEdges(WorkflowSpec const& workflow, // Get the input lifetime and the output lifetime. // Output lifetime must be Timeframe if the input lifetime is Timeframe. bool hasErrors = false; - for (auto& edge : edges) { + for (auto const& edge : edges) { DataProcessorSpec const& producer = workflow[edge.producer]; DataProcessorSpec const& consumer = workflow[edge.consumer]; DataProcessorPoliciesInfo const& producerPolicies = policies[edge.producer]; DataProcessorPoliciesInfo const& consumerPolicies = policies[edge.consumer]; OutputSpec const& output = outputs[edge.outputGlobalIndex]; InputSpec const& input = consumer.inputs[edge.consumerInputIndex]; - for (auto& validator : defaultValidators) { + for (auto const& validator : defaultValidators) { hasErrors |= !validator(errors, producer, output, producerPolicies, consumer, input, consumerPolicies); } } diff --git a/Framework/Core/test/test_Concepts.cxx b/Framework/Core/test/test_Concepts.cxx index 982c748e701e4..375e537cfaec0 100644 --- a/Framework/Core/test/test_Concepts.cxx +++ b/Framework/Core/test/test_Concepts.cxx @@ -87,7 +87,7 @@ TEST_CASE("IdentificationConcepts") REQUIRE(with_originals); - REQUIRE(with_sources>::metadata>); + REQUIRE(with_sources_generator>::metadata>); REQUIRE(with_base_table); diff --git a/Framework/Core/test/test_IndexBuilder.cxx b/Framework/Core/test/test_IndexBuilder.cxx index e357b1164af80..2a273db4333fc 100644 --- a/Framework/Core/test/test_IndexBuilder.cxx +++ b/Framework/Core/test/test_IndexBuilder.cxx @@ -108,7 +108,7 @@ TEST_CASE("TestIndexBuilder") auto t5 = IndexBuilder::materialize(builders1, {t1, t2, t3, t4}, map, schema1, true); // auto t5 = IndexBuilder::materialize({t1, t2, t3, t4}, map, schema1, true); REQUIRE(t5->num_rows() == 4); - IDXs idxt{t5}; + IDXsFrom> idxt{t5}; idxt.bindExternalIndices(&st1, &st2, &st3, &st4); for (auto& row : idxt) { REQUIRE(row.distance().pointId() == row.pointId()); @@ -121,7 +121,7 @@ TEST_CASE("TestIndexBuilder") std::vector builders2; auto t6 = IndexBuilder::materialize(builders2, {t2, t1, t3, t4}, map, schema2, false); REQUIRE(t6->num_rows() == st2.size()); - IDX2s idxs{t6}; + IDX2sFrom> idxs{t6}; std::array fs{0, 1, 2, -1, -1, 4, -1}; std::array cs{0, 1, 2, -1, 5, 6, -1}; idxs.bindExternalIndices(&st1, &st2, &st3, &st4); @@ -222,7 +222,7 @@ TEST_CASE("AdvancedIndexTables") std::vector builders3; auto t3 = IndexBuilder::materialize(builders3, {t1, t2, tc}, map, schema3, false); REQUIRE(t3->num_rows() == st1.size()); - IDX3s idxs{t3}; + IDX3sFrom> idxs{t3}; idxs.bindExternalIndices(&st1, &st2, &st3); count = 0; for (auto const& row : idxs) { diff --git a/Framework/TestWorkflows/src/o2TestMultisource.cxx b/Framework/TestWorkflows/src/o2TestMultisource.cxx index 00bd9ba5093bd..d52a25f67e98f 100644 --- a/Framework/TestWorkflows/src/o2TestMultisource.cxx +++ b/Framework/TestWorkflows/src/o2TestMultisource.cxx @@ -25,8 +25,6 @@ using namespace o2::framework::expressions; namespace o2::aod { O2ORIGIN("EMB"); -template -using BCsFrom = BCs_001From; using TracksPlus = soa::Join; template using TracksPlusFrom = soa::Join, StoredTracksExtra_002From>; @@ -34,9 +32,9 @@ using TracksPlusFrom = soa::Join, StoredTracksExtra_002Fro struct TestEmbeddingSubscription { void process(aod::BCs const& bcs, aod::BCsFrom> const& bcse, - aod::TracksPlus const& tracks, aod::TracksPlusFrom> const& trackse) + aod::TracksIU const& tracks, aod::TracksIUFrom> const& trackse) { - LOGP(info, "BCs from run {} and {}", bcs.begin().runNumber(), bcse.begin().runNumber()); + LOGP(info, "BCs from run {} ({}) and {} ({})", bcs.begin().runNumber(), bcs.size(), bcse.begin().runNumber(), bcse.size()); LOGP(info, "Joined tracks: {} and {}", tracks.size(), trackse.size()); } };