Revert "Use the hash modulo in the derivation outputs"
Fix #11897
As described in the issue, this makes for a simpler and much more
intuitive notion of a realisation key. This is better for pedagogy, and
interoperability between more tools.
The way the issue was written was that we would switch to only having
shallow realisations first, and then do this. But going to only shallow
realisations is more complex change, and it turns out we weren't even
testing for the benefits that derivation hashes (modulo FODs) provided
in the deep realisation case, so I now just want to do this first.
Doing this gets the binary cache data structures in order, which will
unblock the Hydra fixed-output-derivation tracking work. I don't want to
delay that work while I figure out the changes needed for
shallow-realisations only.
This reverts commit bab1cda0e6.
Co-authored-by: Amaan Qureshi <git@amaanq.com>
This commit is contained in:
committed by
Amaan Qureshi
parent
702ebdb11b
commit
4f91e9599f
@@ -83,7 +83,7 @@ properties:
|
||||
description: |
|
||||
A mapping from output names to their build trace entries.
|
||||
additionalProperties:
|
||||
"$ref": "build-trace-entry-v2.yaml"
|
||||
"$ref": "build-trace-entry-v2.yaml#/$defs/value"
|
||||
|
||||
failure:
|
||||
type: object
|
||||
|
||||
@@ -16,24 +16,21 @@ description: |
|
||||
|
||||
- Version 1: Original format
|
||||
|
||||
- Version 2: Remove `dependentRealisations`
|
||||
- Version 2:
|
||||
- Use `drvPath` not `drvHash` to refer to derivation in a more conventional way.
|
||||
- Remove `dependentRealisations`
|
||||
- Separate into `key` and `value`
|
||||
|
||||
type: object
|
||||
required:
|
||||
- id
|
||||
- outPath
|
||||
- signatures
|
||||
allOf:
|
||||
- "$ref": "#/$defs/key"
|
||||
- "$ref": "#/$defs/value"
|
||||
- key
|
||||
- value
|
||||
properties:
|
||||
id: {}
|
||||
outPath: {}
|
||||
signatures: {}
|
||||
additionalProperties:
|
||||
dependentRealisations:
|
||||
description: deprecated field
|
||||
type: object
|
||||
key:
|
||||
"$ref": "#/$defs/key"
|
||||
value:
|
||||
"$ref": "#/$defs/value"
|
||||
additionalProperties: false
|
||||
|
||||
"$defs":
|
||||
key:
|
||||
@@ -43,23 +40,20 @@ additionalProperties:
|
||||
This is the "key" part, refering to a derivation and output.
|
||||
type: object
|
||||
required:
|
||||
- id
|
||||
- drvPath
|
||||
- outputName
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
title: Derivation Output ID
|
||||
pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$"
|
||||
drvPath:
|
||||
"$ref": "store-path-v1.yaml"
|
||||
title: Derivation Path
|
||||
description: |
|
||||
Unique identifier for the derivation output that was built.
|
||||
|
||||
Format: `{hash-quotient-drv}!{output-name}`
|
||||
|
||||
- **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv).
|
||||
Begins with `sha256:`.
|
||||
|
||||
- **output-name**: Name of the specific output (e.g., "out", "dev", "doc")
|
||||
|
||||
Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
|
||||
The store path of the derivation that was built.
|
||||
outputName:
|
||||
type: string
|
||||
title: Output Name
|
||||
description: |
|
||||
Name of the specific output (e.g., "out", "dev", "doc")
|
||||
additionalProperties: false
|
||||
|
||||
value:
|
||||
title: Build Trace Value
|
||||
@@ -77,13 +71,6 @@ additionalProperties:
|
||||
description: |
|
||||
The path to the store object that resulted from building this derivation for the given output name.
|
||||
|
||||
patternProperties:
|
||||
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
|
||||
"$ref": "store-path-v1.yaml"
|
||||
title: Dependent Store Path
|
||||
description: Store path that this dependency resolved to during the build
|
||||
additionalProperties: false
|
||||
|
||||
signatures:
|
||||
type: array
|
||||
title: Build Signatures
|
||||
|
||||
@@ -65,9 +65,6 @@ schemas = [
|
||||
'schema' : schema_dir / 'build-trace-entry-v2.yaml',
|
||||
'files' : [
|
||||
'simple.json',
|
||||
# The field is no longer supported, but we want to show that we
|
||||
# ignore it during parsing.
|
||||
'with-dependent-realisations.json',
|
||||
'with-signature.json',
|
||||
],
|
||||
},
|
||||
|
||||
@@ -108,20 +108,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
|
||||
overloaded{
|
||||
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
|
||||
[&](const BuiltPath::Built & p) {
|
||||
auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
|
||||
for (auto & [outputName, outputPath] : p.outputs) {
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
auto drvOutput = get(drvHashes, outputName);
|
||||
if (!drvOutput)
|
||||
throw Error(
|
||||
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
|
||||
store.printStorePath(p.drvPath->outPath()),
|
||||
outputName);
|
||||
DrvOutput key{*drvOutput, outputName};
|
||||
DrvOutput key{
|
||||
.drvPath = p.drvPath->outPath(),
|
||||
.outputName = outputName,
|
||||
};
|
||||
auto thisRealisation = store.queryRealisation(key);
|
||||
assert(thisRealisation); // We’ve built it, so we must
|
||||
// have the realisation
|
||||
res.insert(Realisation{*thisRealisation, std::move(key)});
|
||||
// We’ve built it, so we must have the realisation.
|
||||
assert(thisRealisation);
|
||||
res.insert(Realisation{*thisRealisation, key});
|
||||
} else {
|
||||
res.insert(outputPath);
|
||||
}
|
||||
|
||||
@@ -75,25 +75,13 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -108,69 +108,6 @@ CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
(std::tuple<DrvOutput, DrvOutput>{
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
realisation,
|
||||
"realisation",
|
||||
(std::tuple<Realisation, Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
READ_CHARACTERIZATION_TEST(
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
(std::tuple<Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
vector,
|
||||
"vector",
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
{
|
||||
"builtOutputs": {
|
||||
"bar": {
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
|
||||
"signatures": []
|
||||
},
|
||||
"foo": {
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": {
|
||||
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
{
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": []
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
|
||||
"outputName": "foo"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"dependentRealisations": {
|
||||
"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"
|
||||
},
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": []
|
||||
}
|
||||
@@ -1,8 +1,12 @@
|
||||
{
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
|
||||
"outputName": "foo"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
BIN
src/libstore-tests/data/serve-protocol/build-result-2.8.bin
Normal file
BIN
src/libstore-tests/data/serve-protocol/build-result-2.8.bin
Normal file
Binary file not shown.
37
src/libstore-tests/data/serve-protocol/build-result-2.8.json
Normal file
37
src/libstore-tests/data/serve-protocol/build-result-2.8.json
Normal file
@@ -0,0 +1,37 @@
|
||||
[
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": false,
|
||||
"startTime": 0,
|
||||
"status": "OutputRejected",
|
||||
"stopTime": 0,
|
||||
"success": false,
|
||||
"timesBuilt": 0
|
||||
},
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": true,
|
||||
"startTime": 30,
|
||||
"status": "NotDeterministic",
|
||||
"stopTime": 50,
|
||||
"success": false,
|
||||
"timesBuilt": 3
|
||||
},
|
||||
{
|
||||
"builtOutputs": {
|
||||
"bar": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
|
||||
"signatures": []
|
||||
},
|
||||
"foo": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"startTime": 30,
|
||||
"status": "Built",
|
||||
"stopTime": 50,
|
||||
"success": true,
|
||||
"timesBuilt": 1
|
||||
}
|
||||
]
|
||||
BIN
src/libstore-tests/data/serve-protocol/drv-output-2.8.bin
Normal file
BIN
src/libstore-tests/data/serve-protocol/drv-output-2.8.bin
Normal file
Binary file not shown.
10
src/libstore-tests/data/serve-protocol/drv-output-2.8.json
Normal file
10
src/libstore-tests/data/serve-protocol/drv-output-2.8.json
Normal file
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "quux"
|
||||
}
|
||||
]
|
||||
BIN
src/libstore-tests/data/serve-protocol/realisation-2.8.bin
Normal file
BIN
src/libstore-tests/data/serve-protocol/realisation-2.8.bin
Normal file
Binary file not shown.
13
src/libstore-tests/data/serve-protocol/realisation-2.8.json
Normal file
13
src/libstore-tests/data/serve-protocol/realisation-2.8.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,39 @@
|
||||
[
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": false,
|
||||
"startTime": 0,
|
||||
"status": "OutputRejected",
|
||||
"stopTime": 0,
|
||||
"success": false,
|
||||
"timesBuilt": 0
|
||||
},
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": true,
|
||||
"startTime": 30,
|
||||
"status": "NotDeterministic",
|
||||
"stopTime": 50,
|
||||
"success": false,
|
||||
"timesBuilt": 3
|
||||
},
|
||||
{
|
||||
"builtOutputs": {
|
||||
"bar": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
|
||||
"signatures": []
|
||||
},
|
||||
"foo": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"cpuSystem": 604000000,
|
||||
"cpuUser": 500000000,
|
||||
"startTime": 30,
|
||||
"status": "Built",
|
||||
"stopTime": 50,
|
||||
"success": true,
|
||||
"timesBuilt": 1
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "quux"
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -37,20 +37,19 @@ TEST(DummyStore, realisation_read)
|
||||
return cfg->openDummyStore();
|
||||
}();
|
||||
|
||||
auto drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16);
|
||||
StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"};
|
||||
|
||||
auto outputName = "foo";
|
||||
|
||||
EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr);
|
||||
EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr);
|
||||
|
||||
UnkeyedRealisation value{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
};
|
||||
|
||||
store->buildTrace.insert({drvHash, {{outputName, value}}});
|
||||
store->buildTrace.insert({drvPath, {{outputName, value}}});
|
||||
|
||||
auto value2 = store->queryRealisation({drvHash, outputName});
|
||||
auto value2 = store->queryRealisation({drvPath, outputName});
|
||||
|
||||
ASSERT_TRUE(value2);
|
||||
EXPECT_EQ(*value2, value);
|
||||
@@ -131,10 +130,7 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] {
|
||||
[&] {
|
||||
auto store = writeCfg->openDummyStore();
|
||||
store->buildTrace.insert_or_assign(
|
||||
Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"},
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
|
||||
@@ -46,13 +46,10 @@ TEST_P(RealisationJsonTest, to_json)
|
||||
|
||||
Realisation simple{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"},
|
||||
.outputName = "foo",
|
||||
},
|
||||
};
|
||||
@@ -77,14 +74,4 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
}(),
|
||||
}));
|
||||
|
||||
/**
|
||||
* We no longer have a notion of "dependent realisations", but we still
|
||||
* want to parse old realisation files. So make this just be a read test
|
||||
* (no write direction), accordingly.
|
||||
*/
|
||||
TEST_F(RealisationTest, dependent_realisations_from_json)
|
||||
{
|
||||
readJsonTest("with-dependent-realisations", simple);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -78,16 +78,19 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
defaultVersion,
|
||||
drvOutput_2_8,
|
||||
"drv-output-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
(std::tuple<DrvOutput, DrvOutput>{
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
@@ -96,54 +99,37 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
realisation,
|
||||
"realisation",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation, Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
unkeyedRealisation_2_8,
|
||||
"unkeyed-realisation-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
(UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
}))
|
||||
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
realisation_2_8,
|
||||
"realisation-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
(Realisation{
|
||||
UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
},
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "baz",
|
||||
},
|
||||
}))
|
||||
|
||||
@@ -209,7 +195,10 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
/* We now do a lossy read which does not allow us to faithfully write
|
||||
back, since we changed the data type. We still however want to test
|
||||
that this read works, and so for that we have a one-way test. */
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
buildResult_2_6,
|
||||
"build-result-2.6",
|
||||
@@ -242,27 +231,72 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
.timesBuilt = 1,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
#if 0
|
||||
// These fields are not yet serialized.
|
||||
// FIXME Include in next version of protocol or document
|
||||
// why they are skipped.
|
||||
.cpuUser = std::chrono::milliseconds(500s),
|
||||
.cpuSystem = std::chrono::milliseconds(604s),
|
||||
#endif
|
||||
},
|
||||
};
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
buildResult_2_8,
|
||||
"build-result-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
({
|
||||
using namespace std::literals::chrono_literals;
|
||||
std::tuple<BuildResult, BuildResult, BuildResult> t{
|
||||
BuildResult{.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::OutputRejected,
|
||||
.msg = HintFmt("no idea why"),
|
||||
}}}},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::NotDeterministic,
|
||||
.msg = HintFmt("no idea why"),
|
||||
.isNonDeterministic = true,
|
||||
}}},
|
||||
.timesBuilt = 3,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Success{
|
||||
.status = BuildResult::Success::Built,
|
||||
.builtOutputs =
|
||||
{
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -18,17 +18,17 @@ namespace nix {
|
||||
TEST(WorkerProtoVersionNumber, ordering)
|
||||
{
|
||||
using Number = WorkerProto::Version::Number;
|
||||
EXPECT_LT((Number{1, 10}), (Number{1, 20}));
|
||||
EXPECT_GT((Number{1, 30}), (Number{1, 20}));
|
||||
EXPECT_EQ((Number{1, 10}), (Number{1, 10}));
|
||||
EXPECT_LT((Number{0, 255}), (Number{1, 0}));
|
||||
EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20}));
|
||||
EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20}));
|
||||
EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10}));
|
||||
EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0}));
|
||||
}
|
||||
|
||||
TEST(WorkerProtoVersion, partialOrderingSameFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
V v1{.number = {1, 20}, .features = {"a", "b"}};
|
||||
V v2{.number = {1, 30}, .features = {"a", "b"}};
|
||||
V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}};
|
||||
V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}};
|
||||
|
||||
EXPECT_TRUE(v1 < v2);
|
||||
EXPECT_TRUE(v2 > v1);
|
||||
@@ -40,8 +40,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures)
|
||||
TEST(WorkerProtoVersion, partialOrderingSubsetFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
V fewer{.number = {1, 30}, .features = {"a"}};
|
||||
V more{.number = {1, 30}, .features = {"a", "b"}};
|
||||
V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}};
|
||||
V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}};
|
||||
|
||||
// fewer <= more: JUST the features are a subset
|
||||
EXPECT_TRUE(fewer < more);
|
||||
@@ -54,8 +54,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
// Same number but incomparable features
|
||||
V v1{.number = {1, 20}, .features = {"a", "c"}};
|
||||
V v2{.number = {1, 20}, .features = {"a", "b"}};
|
||||
V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}};
|
||||
V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}};
|
||||
|
||||
EXPECT_FALSE(v1 < v2);
|
||||
EXPECT_FALSE(v1 > v2);
|
||||
@@ -69,8 +69,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
// Higher number but fewer features — unordered
|
||||
V v1{.number = {1, 30}, .features = {"a"}};
|
||||
V v2{.number = {1, 20}, .features = {"a", "b"}};
|
||||
V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}};
|
||||
V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}};
|
||||
|
||||
EXPECT_FALSE(v1 < v2);
|
||||
EXPECT_FALSE(v1 > v2);
|
||||
@@ -80,8 +80,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
|
||||
TEST(WorkerProtoVersion, partialOrderingEmptyFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
V empty{.number = {1, 20}, .features = {}};
|
||||
V some{.number = {1, 30}, .features = {"a"}};
|
||||
V empty{.number = {.major = 1, .minor = 20}, .features = {}};
|
||||
V some{.number = {.major = 1, .minor = 30}, .features = {"a"}};
|
||||
|
||||
// empty features is a subset of everything
|
||||
EXPECT_TRUE(empty < some);
|
||||
@@ -223,69 +223,67 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
defaultVersion,
|
||||
"drv-output-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
(std::tuple<DrvOutput, DrvOutput>{
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
realisation,
|
||||
"realisation",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation, Realisation>{
|
||||
Realisation{
|
||||
unkeyedRealisation_realisation_with_path,
|
||||
"unkeyed-realisation-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
(UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
}))
|
||||
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation>{
|
||||
Realisation{
|
||||
realisation_realisation_with_path,
|
||||
"realisation-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
(Realisation{
|
||||
UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
},
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "baz",
|
||||
},
|
||||
}))
|
||||
|
||||
@@ -318,7 +316,10 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
/* We now do a lossy read which does not allow us to faithfully write
|
||||
back, since we changed the data type. We still however want to test
|
||||
that this read works, and so for that we have a one-way test. */
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_1_28,
|
||||
"build-result-1.28",
|
||||
@@ -347,25 +348,13 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -374,7 +363,8 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
// See above note
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_1_29,
|
||||
"build-result-1.29",
|
||||
@@ -410,27 +400,13 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -443,7 +419,8 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
// See above note
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_1_37,
|
||||
"build-result-1.37",
|
||||
@@ -479,27 +456,71 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
.timesBuilt = 1,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
.cpuUser = std::chrono::microseconds(500s),
|
||||
.cpuSystem = std::chrono::microseconds(604s),
|
||||
},
|
||||
};
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_realisation_with_path,
|
||||
"build-result-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
({
|
||||
using namespace std::literals::chrono_literals;
|
||||
std::tuple<BuildResult, BuildResult, BuildResult> t{
|
||||
BuildResult{.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::OutputRejected,
|
||||
.msg = HintFmt("no idea why"),
|
||||
}}}},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::NotDeterministic,
|
||||
.msg = HintFmt("no idea why"),
|
||||
.isNonDeterministic = true,
|
||||
}}},
|
||||
.timesBuilt = 3,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Success{
|
||||
.status = BuildResult::Success::Built,
|
||||
.builtOutputs =
|
||||
{
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -920,7 +941,11 @@ TEST_F(WorkerProtoTest, handshake_features)
|
||||
out,
|
||||
in,
|
||||
WorkerProto::Version{
|
||||
.number = {.major = 1, .minor = 123},
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 123,
|
||||
},
|
||||
.features = {"bar", "aap", "mies", "xyzzy"},
|
||||
});
|
||||
});
|
||||
@@ -931,7 +956,11 @@ TEST_F(WorkerProtoTest, handshake_features)
|
||||
out,
|
||||
in,
|
||||
WorkerProto::Version{
|
||||
.number = {.major = 1, .minor = 200},
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 200,
|
||||
},
|
||||
.features = {"foo", "bar", "xyzzy"},
|
||||
});
|
||||
|
||||
|
||||
@@ -198,12 +198,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("ca-drv/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo of the derivation
|
||||
// For CA floating derivations, the kind is Deferred since outputs aren't known until build
|
||||
auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
|
||||
ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto drvHash = hashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object
|
||||
auto outputPath = substituter->addToStore(
|
||||
"test-ca-drv-out",
|
||||
@@ -222,7 +216,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
|
||||
// Add the realisation (build trace) to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
drvHash,
|
||||
drvPath,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
@@ -236,7 +230,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
checkpointJson("ca-drv/substituter", substituter);
|
||||
|
||||
// The realisation should not exist in the destination store yet
|
||||
DrvOutput drvOutput{drvHash, "out"};
|
||||
DrvOutput drvOutput{drvPath, "out"};
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
@@ -299,11 +293,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
// Write the dependency derivation to the destination store
|
||||
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
|
||||
|
||||
// Compute the hash modulo for the dependency derivation
|
||||
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
|
||||
ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto depDrvHash = depHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the dependency in the substituter
|
||||
auto depOutputPath = substituter->addToStore(
|
||||
"dep-drv-out",
|
||||
@@ -322,7 +311,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
|
||||
// Add the realisation for the dependency to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
depDrvHash,
|
||||
depDrvPath,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
@@ -353,11 +342,6 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("issue-11928/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo for the root derivation
|
||||
auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
|
||||
ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto rootDrvHash = rootHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the root derivation
|
||||
// Note: it does NOT reference the dependency's output
|
||||
auto rootOutputPath = substituter->addToStore(
|
||||
@@ -378,12 +362,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// The DrvOutputs for both derivations
|
||||
DrvOutput depDrvOutput{depDrvHash, "out"};
|
||||
DrvOutput rootDrvOutput{rootDrvHash, "out"};
|
||||
DrvOutput depDrvOutput{depDrvPath, "out"};
|
||||
DrvOutput rootDrvOutput{rootDrvPath, "out"};
|
||||
|
||||
// Add the realisation for the root derivation to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
rootDrvHash,
|
||||
rootDrvPath,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
|
||||
@@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore(
|
||||
|
||||
std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id)
|
||||
{
|
||||
return realisationsPrefix + "/" + id.to_string() + ".doi";
|
||||
return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi";
|
||||
}
|
||||
|
||||
void BinaryCacheStore::queryRealisationUncached(
|
||||
@@ -535,7 +535,10 @@ void BinaryCacheStore::queryRealisationUncached(
|
||||
realisation = std::make_shared<const UnkeyedRealisation>(nlohmann::json::parse(*data));
|
||||
} catch (Error & e) {
|
||||
e.addTrace(
|
||||
{}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string());
|
||||
{},
|
||||
"while parsing file '%s' as a build trace value for key '%s'",
|
||||
outputInfoFilePath,
|
||||
id.to_string());
|
||||
throw;
|
||||
}
|
||||
return (*callbackPtr)(std::move(realisation));
|
||||
@@ -551,7 +554,10 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
|
||||
{
|
||||
if (diskCache)
|
||||
diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info);
|
||||
upsertFile(makeRealisationPath(info.id), static_cast<nlohmann::json>(info).dump(), "application/json");
|
||||
upsertFile(
|
||||
makeRealisationPath(info.id),
|
||||
static_cast<nlohmann::json>(static_cast<const UnkeyedRealisation &>(info)).dump(),
|
||||
"application/json");
|
||||
}
|
||||
|
||||
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
|
||||
|
||||
@@ -223,9 +223,8 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
given this information by the downstream goal, that cannot happen
|
||||
anymore if the downstream goal only cares about one output, but
|
||||
we care about all outputs. */
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
for (auto & [outputName, outputHash] : outputHashes) {
|
||||
InitialOutput v{.outputHash = outputHash};
|
||||
for (auto & [outputName, _] : drv->outputs) {
|
||||
InitialOutput v;
|
||||
|
||||
/* TODO we might want to also allow randomizing the paths
|
||||
for regular CA derivations, e.g. for sake of checking
|
||||
@@ -1125,7 +1124,7 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
|
||||
: PathStatus::Corrupt,
|
||||
};
|
||||
}
|
||||
auto drvOutput = DrvOutput{info.outputHash, i.first};
|
||||
auto drvOutput = DrvOutput{drvPath, i.first};
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
if (auto real = worker.store.queryRealisation(drvOutput)) {
|
||||
info.known = {
|
||||
|
||||
@@ -37,12 +37,6 @@ DerivationGoal::DerivationGoal(
|
||||
, drvPath(drvPath)
|
||||
, wantedOutput(wantedOutput)
|
||||
, drv{std::make_unique<Derivation>(drv)}
|
||||
, outputHash{[&] {
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, drv);
|
||||
if (auto * mOutputHash = get(outputHashes, wantedOutput))
|
||||
return *mOutputHash;
|
||||
throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput);
|
||||
}()}
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
|
||||
@@ -102,7 +96,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
them. */
|
||||
if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) {
|
||||
if (!checkResult) {
|
||||
DrvOutput id{outputHash, wantedOutput};
|
||||
DrvOutput id{drvPath, wantedOutput};
|
||||
auto g = worker.makeDrvOutputSubstitutionGoal(id);
|
||||
waitees.insert(g);
|
||||
co_await await(std::move(waitees));
|
||||
@@ -186,12 +180,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
// No `std::visit` for coroutines yet
|
||||
if (auto * successP = resolvedResult.tryGetSuccess()) {
|
||||
auto & success = *successP;
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
|
||||
|
||||
auto outputHash = get(outputHashes, wantedOutput);
|
||||
auto resolvedHash = get(resolvedHashes, wantedOutput);
|
||||
if ((!outputHash) || (!resolvedHash))
|
||||
if (!drv->outputs.contains(wantedOutput))
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)",
|
||||
worker.store.printStorePath(drvPath),
|
||||
@@ -200,7 +189,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
auto realisation = [&] {
|
||||
auto take1 = get(success.builtOutputs, wantedOutput);
|
||||
if (take1)
|
||||
return static_cast<UnkeyedRealisation>(*take1);
|
||||
return *take1;
|
||||
|
||||
/* The above `get` should work. But stateful tracking of
|
||||
outputs in resolvedResult, this can get out of sync with the
|
||||
@@ -208,7 +197,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
check the store directly if it fails. */
|
||||
auto take2 = worker.evalStore.queryRealisation(
|
||||
DrvOutput{
|
||||
.drvHash = *resolvedHash,
|
||||
.drvPath = pathResolved,
|
||||
.outputName = wantedOutput,
|
||||
});
|
||||
if (take2)
|
||||
@@ -224,7 +213,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
Realisation newRealisation{
|
||||
realisation,
|
||||
{
|
||||
.drvHash = *outputHash,
|
||||
.drvPath = drvPath,
|
||||
.outputName = wantedOutput,
|
||||
}};
|
||||
newRealisation.signatures.clear();
|
||||
@@ -270,16 +259,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
/* In checking mode, the builder will not register any outputs.
|
||||
So we want to make sure the ones that we wanted to check are
|
||||
properly there. */
|
||||
success.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
assertPathValidity(),
|
||||
{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}};
|
||||
success.builtOutputs = {{wantedOutput, assertPathValidity()}};
|
||||
} else {
|
||||
/* Otherwise the builder will give us info for out output, but
|
||||
also for other outputs. Filter down to just our output so as
|
||||
@@ -298,16 +278,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
if (success.builtOutputs.count(wantedOutput) == 0) {
|
||||
debug(
|
||||
"BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput);
|
||||
success.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
assertPathValidity(),
|
||||
{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}};
|
||||
success.builtOutputs = {{wantedOutput, assertPathValidity()}};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -404,7 +375,7 @@ std::optional<std::pair<UnkeyedRealisation, PathStatus>> DerivationGoal::checkPa
|
||||
if (drv->type().isImpure())
|
||||
return std::nullopt;
|
||||
|
||||
auto drvOutput = DrvOutput{outputHash, wantedOutput};
|
||||
auto drvOutput = DrvOutput{drvPath, wantedOutput};
|
||||
|
||||
std::optional<UnkeyedRealisation> mRealisation;
|
||||
|
||||
@@ -444,7 +415,7 @@ std::optional<std::pair<UnkeyedRealisation, PathStatus>> DerivationGoal::checkPa
|
||||
Realisation{
|
||||
*mRealisation,
|
||||
{
|
||||
.drvHash = outputHash,
|
||||
.drvPath = drvPath,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
});
|
||||
@@ -475,16 +446,7 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
.builtOutputs = {{wantedOutput, std::move(builtOutput)}},
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke
|
||||
: Goal(worker, init())
|
||||
, id(id)
|
||||
{
|
||||
name = fmt("substitution of '%s'", id.to_string());
|
||||
name = fmt("substitution of '%s'", id.render(worker.store));
|
||||
trace("created");
|
||||
}
|
||||
|
||||
@@ -93,7 +93,8 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
debug(
|
||||
"derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store));
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
@@ -108,7 +109,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
|
||||
std::string DrvOutputSubstitutionGoal::key()
|
||||
{
|
||||
return "a$" + std::string(id.to_string());
|
||||
return "a$" + std::string(id.render(worker.store));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -2,7 +2,16 @@
|
||||
-- Won't be loaded unless the experimental feature `ca-derivations`
|
||||
-- is enabled
|
||||
|
||||
create table if not exists Realisations (
|
||||
-- Why the `*V<N>` tables
|
||||
--
|
||||
-- We are trying to keep different versions of the experiment to have
|
||||
-- completely independent extra schemas from one another. This will
|
||||
-- enable people to switch between versions of the experiment (including
|
||||
-- newer to older) without migrating between them, but at the cost
|
||||
-- of having many abandoned tables lying around. Closer to the end of
|
||||
-- the experiment, we'll provide guidance on how to clean this up.
|
||||
|
||||
create table if not exists BuildTraceV2 (
|
||||
id integer primary key autoincrement not null,
|
||||
drvPath text not null,
|
||||
outputName text not null, -- symbolic output id, usually "out"
|
||||
@@ -11,31 +20,4 @@ create table if not exists Realisations (
|
||||
foreign key (outputPath) references ValidPaths(id) on delete cascade
|
||||
);
|
||||
|
||||
create index if not exists IndexRealisations on Realisations(drvPath, outputName);
|
||||
|
||||
-- We can end-up in a weird edge-case where a path depends on itself because
|
||||
-- it’s an output of a CA derivation, that happens to be the same as one of its
|
||||
-- dependencies.
|
||||
-- In that case we have a dependency loop (path -> realisation1 -> realisation2
|
||||
-- -> path) that we need to break by removing the dependencies between the
|
||||
-- realisations
|
||||
create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths
|
||||
begin
|
||||
delete from RealisationsRefs where realisationReference in (
|
||||
select id from Realisations where outputPath = old.id
|
||||
);
|
||||
end;
|
||||
|
||||
create table if not exists RealisationsRefs (
|
||||
referrer integer not null,
|
||||
realisationReference integer,
|
||||
foreign key (referrer) references Realisations(id) on delete cascade,
|
||||
foreign key (realisationReference) references Realisations(id) on delete restrict
|
||||
);
|
||||
-- used by deletion trigger
|
||||
create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference);
|
||||
|
||||
-- used by QueryRealisationReferences
|
||||
create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
|
||||
-- used by cascade deletion when ValidPaths is deleted
|
||||
create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);
|
||||
create index if not exists IndexBuildTraceV2 on BuildTraceV2(drvPath, outputName);
|
||||
|
||||
@@ -47,34 +47,6 @@ void CommonProto::Serialise<ContentAddress>::write(
|
||||
conn.to << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
std::string rawInput = readString(conn.from);
|
||||
try {
|
||||
return nlohmann::json::parse(rawInput);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing a realisation object in the remote protocol");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<Realisation>::write(
|
||||
const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
|
||||
{
|
||||
conn.to << static_cast<nlohmann::json>(realisation).dump();
|
||||
}
|
||||
|
||||
DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return DrvOutput::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<DrvOutput>::write(
|
||||
const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
|
||||
{
|
||||
conn.to << drvOutput.to_string();
|
||||
}
|
||||
|
||||
std::optional<StorePath>
|
||||
CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
|
||||
@@ -955,14 +955,8 @@ static void performOp(
|
||||
|
||||
case WorkerProto::Op::RegisterDrvOutput: {
|
||||
logger->startWork();
|
||||
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
|
||||
auto outputPath = StorePath(readString(conn.from));
|
||||
store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId});
|
||||
} else {
|
||||
auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
|
||||
store->registerDrvOutput(realisation);
|
||||
}
|
||||
auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
|
||||
store->registerDrvOutput(realisation);
|
||||
logger->stopWork();
|
||||
break;
|
||||
}
|
||||
@@ -970,19 +964,13 @@ static void performOp(
|
||||
case WorkerProto::Op::QueryRealisation: {
|
||||
logger->startWork();
|
||||
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
|
||||
auto info = store->queryRealisation(outputId);
|
||||
std::optional<UnkeyedRealisation> info = *store->queryRealisation(outputId);
|
||||
logger->stopWork();
|
||||
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
std::set<StorePath> outPaths;
|
||||
if (info)
|
||||
outPaths.insert(info->outPath);
|
||||
WorkerProto::write(*store, wconn, outPaths);
|
||||
} else {
|
||||
std::set<Realisation> realisations;
|
||||
if (info)
|
||||
realisations.insert({*info, outputId});
|
||||
WorkerProto::write(*store, wconn, realisations);
|
||||
}
|
||||
/* Only return the new format because if we got past
|
||||
`DrvOutput` serialization, we know that is what we're using.
|
||||
*/
|
||||
assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath));
|
||||
WorkerProto::write(*store, wconn, info);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -858,13 +858,14 @@ DrvHashes drvHashes;
|
||||
/* Look up the derivation by value and memoize the
|
||||
`hashDerivationModulo` call.
|
||||
*/
|
||||
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||
static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||
{
|
||||
std::optional<DrvHash> hash;
|
||||
std::optional<DrvHashModulo> hash;
|
||||
if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) {
|
||||
return *hash;
|
||||
}
|
||||
auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false);
|
||||
|
||||
// Cache it
|
||||
drvHashes.insert_or_assign(drvPath, h);
|
||||
return h;
|
||||
@@ -887,12 +888,10 @@ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPa
|
||||
don't leak the provenance of fixed outputs, reducing pointless cache
|
||||
misses as the build itself won't know this.
|
||||
*/
|
||||
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
{
|
||||
auto type = drv.type();
|
||||
|
||||
/* Return a fixed hash for fixed-output derivations. */
|
||||
if (type.isFixed()) {
|
||||
if (drv.type().isFixed()) {
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & i : drv.outputs) {
|
||||
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
|
||||
@@ -902,54 +901,66 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
|
||||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||
}
|
||||
return DrvHash{
|
||||
.hashes = outputHashes,
|
||||
.kind = DrvHash::Kind::Regular,
|
||||
};
|
||||
return outputHashes;
|
||||
}
|
||||
|
||||
auto kind = std::visit(
|
||||
overloaded{
|
||||
[](const DerivationType::InputAddressed & ia) {
|
||||
/* This might be a "pesimistically" deferred output, so we don't
|
||||
"taint" the kind yet. */
|
||||
return DrvHash::Kind::Regular;
|
||||
},
|
||||
[](const DerivationType::ContentAddressed & ca) {
|
||||
return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred;
|
||||
},
|
||||
[](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }},
|
||||
drv.type().raw);
|
||||
if (std::visit(
|
||||
overloaded{
|
||||
[](const DerivationType::InputAddressed & ia) {
|
||||
/* This might be a "pesimistically" deferred output, so we don't
|
||||
"taint" the kind yet. */
|
||||
return false;
|
||||
},
|
||||
[](const DerivationType::ContentAddressed & ca) {
|
||||
// Already covered
|
||||
assert(!ca.fixed);
|
||||
return true;
|
||||
},
|
||||
[](const DerivationType::Impure &) { return true; }},
|
||||
drv.type().raw)) {
|
||||
return DrvHashModulo::DeferredDrv{};
|
||||
}
|
||||
|
||||
/* For other derivations, replace the inputs paths with recursive
|
||||
calls to this function. */
|
||||
DerivedPathMap<StringSet>::ChildNode::Map inputs2;
|
||||
for (auto & [drvPath, node] : drv.inputDrvs.map) {
|
||||
/* Need to build and resolve dynamic derivations first */
|
||||
if (!node.childMap.empty()) {
|
||||
return DrvHashModulo::DeferredDrv{};
|
||||
}
|
||||
|
||||
const auto & res = pathDerivationModulo(store, drvPath);
|
||||
if (res.kind == DrvHash::Kind::Deferred)
|
||||
kind = DrvHash::Kind::Deferred;
|
||||
for (auto & outputName : node.value) {
|
||||
const auto h = get(res.hashes, outputName);
|
||||
if (!h)
|
||||
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
|
||||
inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName);
|
||||
if (std::visit(
|
||||
overloaded{
|
||||
[&](const DrvHashModulo::DeferredDrv &) { return true; },
|
||||
// Regular non-CA derivation, replace derivation
|
||||
[&](const DrvHashModulo::DrvHash & drvHash) {
|
||||
inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node);
|
||||
return false;
|
||||
},
|
||||
// CA derivation's output hashes
|
||||
[&](const DrvHashModulo::CaOutputHashes & outputHashes) {
|
||||
for (auto & outputName : node.value) {
|
||||
/* Put each one in with a single "out" output.. */
|
||||
const auto h = get(outputHashes, outputName);
|
||||
if (!h)
|
||||
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
|
||||
inputs2.insert_or_assign(
|
||||
h->to_string(HashFormat::Base16, false),
|
||||
DerivedPathMap<StringSet>::ChildNode{
|
||||
.value = {"out"},
|
||||
});
|
||||
}
|
||||
return false;
|
||||
},
|
||||
},
|
||||
res.raw)) {
|
||||
return DrvHashModulo::DeferredDrv{};
|
||||
}
|
||||
}
|
||||
|
||||
auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & [outputName, _] : drv.outputs) {
|
||||
outputHashes.insert_or_assign(outputName, hash);
|
||||
}
|
||||
|
||||
return DrvHash{
|
||||
.hashes = outputHashes,
|
||||
.kind = kind,
|
||||
};
|
||||
}
|
||||
|
||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
|
||||
{
|
||||
return hashDerivationModulo(store, drv, true).hashes;
|
||||
return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
}
|
||||
|
||||
static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
|
||||
@@ -1099,26 +1110,6 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites)
|
||||
}
|
||||
}
|
||||
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||
{
|
||||
drv.applyRewrites(rewrites);
|
||||
|
||||
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
|
||||
for (auto & [outputName, output] : drv.outputs) {
|
||||
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) {
|
||||
auto h = get(hashModulo.hashes, outputName);
|
||||
if (!h)
|
||||
throw Error(
|
||||
"derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName);
|
||||
auto outPath = store.makeOutputPath(outputName, *h, drv.name);
|
||||
drv.env[outputName] = store.printStorePath(outPath);
|
||||
output = DerivationOutput::InputAddressed{
|
||||
.path = std::move(outPath),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Derivation::shouldResolve() const
|
||||
{
|
||||
/* No input drvs means nothing to resolve. */
|
||||
@@ -1230,9 +1221,13 @@ std::optional<BasicDerivation> Derivation::tryResolve(
|
||||
queryResolutionChain))
|
||||
return std::nullopt;
|
||||
|
||||
rewriteDerivation(store, resolved, inputRewrites);
|
||||
resolved.applyRewrites(inputRewrites);
|
||||
|
||||
return resolved;
|
||||
Derivation resolved2{std::move(resolved)};
|
||||
|
||||
resolved2.fillInOutputPaths(store);
|
||||
|
||||
return resolved2;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1259,7 +1254,15 @@ std::optional<BasicDerivation> Derivation::tryResolve(
|
||||
template<bool fillIn>
|
||||
static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName)
|
||||
{
|
||||
std::optional<DrvHash> hashesModulo;
|
||||
std::optional<DrvHashModulo> hashModulo_;
|
||||
|
||||
auto hashModulo = [&]() -> const auto & {
|
||||
if (!hashModulo_) {
|
||||
// somewhat expensive so we do lazily
|
||||
hashModulo_ = hashDerivationModulo(store, drv, true);
|
||||
}
|
||||
return *hashModulo_;
|
||||
};
|
||||
|
||||
for (auto & [outputName, output] : drv.outputs) {
|
||||
auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) {
|
||||
@@ -1296,65 +1299,64 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string
|
||||
}
|
||||
};
|
||||
auto hash = [&]<typename Output>(const Output & outputVariant) {
|
||||
if (!hashesModulo) {
|
||||
// somewhat expensive so we do lazily
|
||||
hashesModulo = hashDerivationModulo(store, drv, true);
|
||||
}
|
||||
switch (hashesModulo->kind) {
|
||||
case DrvHash::Kind::Regular: {
|
||||
auto h = get(hashesModulo->hashes, outputName);
|
||||
if (!h)
|
||||
throw Error("derivation produced no hash for output '%s'", outputName);
|
||||
auto outPath = store.makeOutputPath(outputName, *h, drvName);
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const DrvHashModulo::DrvHash & drvHash) {
|
||||
auto outPath = store.makeOutputPath(outputName, drvHash, drvName);
|
||||
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
if (outputVariant.path == outPath) {
|
||||
return; // Correct case
|
||||
}
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be '%s'",
|
||||
store.printStorePath(outputVariant.path),
|
||||
store.printStorePath(outPath));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
if constexpr (fillIn)
|
||||
/* Fill in output path for Deferred
|
||||
outputs */
|
||||
output = DerivationOutput::InputAddressed{
|
||||
.path = outPath,
|
||||
};
|
||||
else
|
||||
/* Validation mode: deferred outputs
|
||||
should have been filled in */
|
||||
warn(
|
||||
"derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.",
|
||||
store.printStorePath(outPath));
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
envHasRightPath(outPath);
|
||||
break;
|
||||
}
|
||||
case DrvHash::Kind::Deferred:
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be deferred",
|
||||
store.printStorePath(outputVariant.path));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
/* Correct: Deferred output with Deferred
|
||||
hash kind. */
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
break;
|
||||
}
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
if (outputVariant.path == outPath) {
|
||||
envHasRightPath(outPath);
|
||||
return; // Correct case
|
||||
}
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be '%s'",
|
||||
store.printStorePath(outputVariant.path),
|
||||
store.printStorePath(outPath));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
if constexpr (fillIn) {
|
||||
/* Fill in output path for Deferred outputs */
|
||||
output = DerivationOutput::InputAddressed{
|
||||
.path = outPath,
|
||||
};
|
||||
envHasRightPath(outPath);
|
||||
} else {
|
||||
/* Validation mode: deferred outputs
|
||||
should have been filled in */
|
||||
warn(
|
||||
"derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.",
|
||||
store.printStorePath(outPath));
|
||||
}
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
},
|
||||
[&](const DrvHashModulo::CaOutputHashes &) {
|
||||
/* Shouldn't happen as the original output is
|
||||
input-addressed (or deferred waiting to be). */
|
||||
assert(false);
|
||||
},
|
||||
[&](const DrvHashModulo::DeferredDrv &) {
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be deferred",
|
||||
store.printStorePath(outputVariant.path));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
/* Correct: Deferred output with Deferred hash kind. */
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
},
|
||||
},
|
||||
hashModulo().raw);
|
||||
};
|
||||
std::visit(
|
||||
overloaded{
|
||||
|
||||
@@ -330,7 +330,7 @@ struct DummyStoreImpl : DummyStore
|
||||
|
||||
void registerDrvOutput(const Realisation & output) override
|
||||
{
|
||||
buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) {
|
||||
buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) {
|
||||
kv.second.insert_or_assign(output.id.outputName, output);
|
||||
});
|
||||
}
|
||||
@@ -339,7 +339,7 @@ struct DummyStoreImpl : DummyStore
|
||||
const DrvOutput & drvOutput, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
|
||||
{
|
||||
bool visited = false;
|
||||
buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) {
|
||||
buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) {
|
||||
if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) {
|
||||
visited = true;
|
||||
callback(std::make_shared<UnkeyedRealisation>(it->second));
|
||||
@@ -436,11 +436,7 @@ ref<DummyStore> adl_serializer<ref<DummyStore>>::from_json(const json & json)
|
||||
for (auto & [k1, v2] : getObject(v)) {
|
||||
UnkeyedRealisation realisation = v2;
|
||||
res->buildTrace.insert_or_visit(
|
||||
{
|
||||
Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64),
|
||||
{{k1, realisation}},
|
||||
},
|
||||
[&](auto & kv) { kv.second.insert_or_assign(k1, realisation); });
|
||||
{StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); });
|
||||
}
|
||||
}
|
||||
return res;
|
||||
@@ -473,7 +469,7 @@ void adl_serializer<DummyStore>::to_json(json & json, const DummyStore & val)
|
||||
auto obj = json::object();
|
||||
val.buildTrace.cvisit_all([&](const auto & kv) {
|
||||
auto & [k, v] = kv;
|
||||
auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object();
|
||||
auto & obj2 = obj[k.to_string()] = json::object();
|
||||
for (auto & [k2, v2] : kv.second)
|
||||
obj2[k2] = v2;
|
||||
});
|
||||
|
||||
@@ -90,8 +90,18 @@ protected:
|
||||
|
||||
/**
|
||||
* The prefix under which realisation infos will be stored
|
||||
*
|
||||
* @note The previous (still experimental, though) hash-keyed
|
||||
* realisations were under "realisations". "build trace" is a better
|
||||
* name anyways (issue #11895). This is call "v2" accordingly.
|
||||
*
|
||||
* While we're experimenting, we'll freely increase this version
|
||||
* number. Old build traces will just be "abandoned" at the old URL.
|
||||
* When we are done experimenting, we'll try lean more on versioning
|
||||
* the build trace entries themselves than the entire directory, for
|
||||
* a smoother migration path.
|
||||
*/
|
||||
constexpr const static std::string realisationsPrefix = "realisations";
|
||||
constexpr const static std::string realisationsPrefix = "build-trace-v2";
|
||||
|
||||
constexpr const static std::string cacheInfoFile = "nix-cache-info";
|
||||
|
||||
@@ -100,7 +110,7 @@ protected:
|
||||
/**
|
||||
* Compute the path to the given realisation
|
||||
*
|
||||
* It's `${realisationsPrefix}/${drvOutput}.doi`.
|
||||
* It's `${realisationsPrefix}/${drvPath}/${outputName}`.
|
||||
*/
|
||||
std::string makeRealisationPath(const DrvOutput & id);
|
||||
|
||||
|
||||
@@ -45,7 +45,6 @@ struct InitialOutputStatus
|
||||
|
||||
struct InitialOutput
|
||||
{
|
||||
Hash outputHash;
|
||||
std::optional<InitialOutputStatus> known;
|
||||
};
|
||||
|
||||
|
||||
@@ -66,8 +66,6 @@ private:
|
||||
*/
|
||||
std::unique_ptr<Derivation> drv;
|
||||
|
||||
const Hash outputHash;
|
||||
|
||||
const BuildMode buildMode;
|
||||
|
||||
/**
|
||||
|
||||
@@ -26,12 +26,13 @@ namespace nix {
|
||||
LengthPrefixedProtoHelper<CommonProto, T>::write(store, conn, t); \
|
||||
}
|
||||
|
||||
#define COMMA_ ,
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
|
||||
#define COMMA_ ,
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T COMMA_ typename Compare>, std::set<T COMMA_ Compare>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename K COMMA_ typename V>, std::map<K COMMA_ V>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K COMMA_ typename V COMMA_ typename Compare>, std::map<K COMMA_ V COMMA_ Compare>)
|
||||
#undef COMMA_
|
||||
|
||||
/* protocol-specific templates */
|
||||
|
||||
@@ -88,8 +88,9 @@ DECLARE_COMMON_SERIALISER(std::set<T COMMA_ Compare>);
|
||||
template<typename... Ts>
|
||||
DECLARE_COMMON_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
template<typename K, typename V>
|
||||
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V>);
|
||||
template<typename K, typename V, typename Compare>
|
||||
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
|
||||
#undef COMMA_
|
||||
|
||||
/**
|
||||
* These use the empty string for the null case, relying on the fact
|
||||
|
||||
@@ -502,34 +502,39 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName);
|
||||
* derivations (fixed-output or not) will have a different hash for each
|
||||
* output.
|
||||
*/
|
||||
struct DrvHash
|
||||
struct DrvHashModulo
|
||||
{
|
||||
/**
|
||||
* Map from output names to hashes
|
||||
* Single hash for the derivation
|
||||
*
|
||||
* This is for an input-addressed derivation that doesn't
|
||||
* transitively depend on any floating-CA derivations.
|
||||
*/
|
||||
std::map<std::string, Hash> hashes;
|
||||
|
||||
enum struct Kind : bool {
|
||||
/**
|
||||
* Statically determined derivations.
|
||||
* This hash will be directly used to compute the output paths
|
||||
*/
|
||||
Regular,
|
||||
|
||||
/**
|
||||
* Floating-output derivations (and their reverse dependencies).
|
||||
*/
|
||||
Deferred,
|
||||
};
|
||||
using DrvHash = Hash;
|
||||
|
||||
/**
|
||||
* The kind of derivation this is, simplified for just "derivation hash
|
||||
* modulo" purposes.
|
||||
* Known CA drv's output hashes, for fixed-output derivations whose
|
||||
* output hashes are always known since they are fixed up-front.
|
||||
*/
|
||||
Kind kind;
|
||||
};
|
||||
using CaOutputHashes = std::map<std::string, Hash>;
|
||||
|
||||
void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
|
||||
/**
|
||||
* This derivation doesn't yet have known output hashes.
|
||||
*
|
||||
* Either because itself is floating CA, or it (transtively) depends
|
||||
* on a floating CA derivation.
|
||||
*/
|
||||
using DeferredDrv = std::monostate;
|
||||
|
||||
using Raw = std::variant<DrvHash, CaOutputHashes, DeferredDrv>;
|
||||
|
||||
Raw raw;
|
||||
|
||||
bool operator==(const DrvHashModulo &) const = default;
|
||||
// auto operator <=> (const DrvHashModulo &) const = default;
|
||||
|
||||
MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo);
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns hashes with the details of fixed-output subderivations
|
||||
@@ -555,15 +560,17 @@ void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
|
||||
* ATerm, after subderivations have been likewise expunged from that
|
||||
* derivation.
|
||||
*/
|
||||
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
|
||||
/**
|
||||
* Return a map associating each output to a hash that uniquely identifies its
|
||||
* derivation (modulo the self-references).
|
||||
* If a derivation is input addressed and doesn't yet have its input
|
||||
* addressed (is deferred) try using `hashDerivationModulo`.
|
||||
*
|
||||
* \todo What is the Hash in this map?
|
||||
* Does nothing if not deferred input-addressed, or
|
||||
* `hashDerivationModulo` indicates it is missing inputs' output paths
|
||||
* and is not yet ready (and must stay deferred).
|
||||
*/
|
||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
|
||||
void resolveInputAddressed(Store & store, Derivation & drv);
|
||||
|
||||
struct DrvHashFct
|
||||
{
|
||||
@@ -578,7 +585,7 @@ struct DrvHashFct
|
||||
/**
|
||||
* Memoisation of hashDerivationModulo().
|
||||
*/
|
||||
typedef boost::concurrent_flat_map<StorePath, DrvHash, DrvHashFct> DrvHashes;
|
||||
typedef boost::concurrent_flat_map<StorePath, DrvHashModulo, DrvHashFct> DrvHashes;
|
||||
|
||||
// FIXME: global, though at least thread-safe.
|
||||
extern DrvHashes drvHashes;
|
||||
|
||||
@@ -49,7 +49,7 @@ struct DummyStore : virtual Store
|
||||
* outer map for the derivation, and inner maps for the outputs of a
|
||||
* given derivation.
|
||||
*/
|
||||
boost::concurrent_flat_map<Hash, std::map<std::string, UnkeyedRealisation>> buildTrace;
|
||||
boost::concurrent_flat_map<StorePath, std::map<std::string, UnkeyedRealisation>> buildTrace;
|
||||
|
||||
DummyStore(ref<const Config> config)
|
||||
: Store{*config}
|
||||
|
||||
@@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector<T>);
|
||||
#define COMMA_ ,
|
||||
template<class Inner, typename T, typename Compare>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T COMMA_ Compare>);
|
||||
#undef COMMA_
|
||||
|
||||
template<class Inner, typename... Ts>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>);
|
||||
|
||||
template<class Inner, typename K, typename V>
|
||||
#define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V>
|
||||
template<class Inner, typename K, typename V, typename Compare>
|
||||
#define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V, Compare>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X);
|
||||
#undef COMMA_
|
||||
|
||||
template<class Inner, typename T>
|
||||
std::vector<T>
|
||||
@@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper<Inner, std::set<T, Compare>>::write(
|
||||
}
|
||||
}
|
||||
|
||||
template<class Inner, typename K, typename V>
|
||||
std::map<K, V>
|
||||
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(const StoreDirConfig & store, typename Inner::ReadConn conn)
|
||||
template<class Inner, typename K, typename V, typename Compare>
|
||||
std::map<K, V, Compare> LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::read(
|
||||
const StoreDirConfig & store, typename Inner::ReadConn conn)
|
||||
{
|
||||
std::map<K, V> resMap;
|
||||
std::map<K, V, Compare> resMap;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
auto k = S<K>::read(store, conn);
|
||||
@@ -123,9 +123,9 @@ LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(const StoreDirConfig & st
|
||||
return resMap;
|
||||
}
|
||||
|
||||
template<class Inner, typename K, typename V>
|
||||
void LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
|
||||
const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
|
||||
template<class Inner, typename K, typename V, typename Compare>
|
||||
void LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::write(
|
||||
const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V, Compare> & resMap)
|
||||
{
|
||||
conn.to << resMap.size();
|
||||
for (auto & i : resMap) {
|
||||
|
||||
@@ -18,33 +18,40 @@ struct OutputsSpec;
|
||||
/**
|
||||
* A general `Realisation` key.
|
||||
*
|
||||
* This is similar to a `DerivedPath::Opaque`, but the derivation is
|
||||
* identified by its "hash modulo" instead of by its store path.
|
||||
* This is similar to a `DerivedPath::Built`, except it is only a single
|
||||
* step: `drvPath` is a `StorePath` rather than a `DerivedPath`.
|
||||
*/
|
||||
struct DrvOutput
|
||||
{
|
||||
/**
|
||||
* The hash modulo of the derivation.
|
||||
*
|
||||
* Computed from the derivation itself for most types of
|
||||
* derivations, but computed from the (fixed) content address of the
|
||||
* output for fixed-output derivations.
|
||||
* The store path to the derivation
|
||||
*/
|
||||
Hash drvHash;
|
||||
StorePath drvPath;
|
||||
|
||||
/**
|
||||
* The name of the output.
|
||||
*/
|
||||
OutputName outputName;
|
||||
|
||||
/**
|
||||
* Skips the store dir on the `drvPath`
|
||||
*/
|
||||
std::string to_string() const;
|
||||
|
||||
std::string strHash() const
|
||||
{
|
||||
return drvHash.to_string(HashFormat::Base16, true);
|
||||
}
|
||||
/**
|
||||
* Skips the store dir on the `drvPath`
|
||||
*/
|
||||
static DrvOutput from_string(std::string_view);
|
||||
|
||||
static DrvOutput parse(const std::string &);
|
||||
/**
|
||||
* Includes the store dir on `drvPath`
|
||||
*/
|
||||
std::string render(const StoreDirConfig & store) const;
|
||||
|
||||
/**
|
||||
* Includes the store dir on `drvPath`
|
||||
*/
|
||||
static DrvOutput parse(const StoreDirConfig & store, std::string_view);
|
||||
|
||||
bool operator==(const DrvOutput &) const = default;
|
||||
auto operator<=>(const DrvOutput &) const = default;
|
||||
@@ -64,6 +71,16 @@ struct UnkeyedRealisation
|
||||
|
||||
size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const;
|
||||
|
||||
/**
|
||||
* Just check the `outPath`. Signatures don't matter for this.
|
||||
* Callers must ensure that the corresponding key is the same for
|
||||
* most use-cases.
|
||||
*/
|
||||
bool isCompatibleWith(const UnkeyedRealisation & other) const
|
||||
{
|
||||
return outPath == other.outPath;
|
||||
}
|
||||
|
||||
const StorePath & getPath() const
|
||||
{
|
||||
return outPath;
|
||||
@@ -77,8 +94,6 @@ struct Realisation : UnkeyedRealisation
|
||||
{
|
||||
DrvOutput id;
|
||||
|
||||
bool isCompatibleWith(const UnkeyedRealisation & other) const;
|
||||
|
||||
bool operator==(const Realisation &) const = default;
|
||||
auto operator<=>(const Realisation &) const = default;
|
||||
};
|
||||
@@ -89,16 +104,7 @@ struct Realisation : UnkeyedRealisation
|
||||
* Since these are the outputs of a single derivation, we know the
|
||||
* output names are unique so we can use them as the map key.
|
||||
*/
|
||||
typedef std::map<OutputName, Realisation> SingleDrvOutputs;
|
||||
|
||||
/**
|
||||
* Collection type for multiple derivations' outputs' `Realisation`s.
|
||||
*
|
||||
* `DrvOutput` is used because in general the derivations are not all
|
||||
* the same, so we need to identify firstly which derivation, and
|
||||
* secondly which output of that derivation.
|
||||
*/
|
||||
typedef std::map<DrvOutput, Realisation> DrvOutputs;
|
||||
typedef std::map<OutputName, UnkeyedRealisation> SingleDrvOutputs;
|
||||
|
||||
struct OpaquePath
|
||||
{
|
||||
@@ -149,19 +155,17 @@ struct RealisedPath
|
||||
class MissingRealisation : public Error
|
||||
{
|
||||
public:
|
||||
MissingRealisation(DrvOutput & outputId)
|
||||
: MissingRealisation(outputId.outputName, outputId.strHash())
|
||||
MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId)
|
||||
: MissingRealisation(store, outputId.drvPath, outputId.outputName)
|
||||
{
|
||||
}
|
||||
|
||||
MissingRealisation(std::string_view drv, OutputName outputName)
|
||||
: Error(
|
||||
"cannot operate on output '%s' of the "
|
||||
"unbuilt derivation '%s'",
|
||||
outputName,
|
||||
drv)
|
||||
{
|
||||
}
|
||||
MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName);
|
||||
MissingRealisation(
|
||||
const StoreDirConfig & store,
|
||||
const SingleDerivedPath & drvPath,
|
||||
const StorePath & drvPathResolved,
|
||||
const OutputName & outputName);
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -34,8 +34,10 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
#define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
|
||||
SERVE_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>,
|
||||
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V>)
|
||||
template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA
|
||||
typename Compare>
|
||||
,
|
||||
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
|
||||
|
||||
/**
|
||||
* Use `CommonProto` where possible.
|
||||
|
||||
@@ -8,12 +8,18 @@ namespace nix {
|
||||
#define SERVE_MAGIC_1 0x390c9deb
|
||||
#define SERVE_MAGIC_2 0x5452eecb
|
||||
|
||||
#define SERVE_PROTOCOL_VERSION (2 << 8 | 8)
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
struct StoreDirConfig;
|
||||
struct Source;
|
||||
|
||||
// items being serialised
|
||||
struct BuildResult;
|
||||
struct UnkeyedValidPathInfo;
|
||||
struct DrvOutput;
|
||||
struct UnkeyedRealisation;
|
||||
struct Realisation;
|
||||
|
||||
/**
|
||||
* The "serve protocol", used by ssh:// stores.
|
||||
@@ -63,7 +69,7 @@ struct ServeProto
|
||||
|
||||
static constexpr Version latest = {
|
||||
.major = 2,
|
||||
.minor = 7,
|
||||
.minor = 8,
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -205,6 +211,12 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op)
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(BuildResult);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(DrvOutput);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(UnkeyedRealisation);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(Realisation);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions);
|
||||
@@ -217,8 +229,8 @@ DECLARE_SERVE_SERIALISER(std::set<T COMMA_ Compare>);
|
||||
template<typename... Ts>
|
||||
DECLARE_SERVE_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
template<typename K, typename V>
|
||||
DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V>);
|
||||
template<typename K, typename V, typename Compare>
|
||||
DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
|
||||
#undef COMMA_
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -34,8 +34,10 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
#define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
|
||||
WORKER_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>,
|
||||
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V>)
|
||||
template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA
|
||||
typename Compare>
|
||||
,
|
||||
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
|
||||
|
||||
/**
|
||||
* Use `CommonProto` where possible.
|
||||
|
||||
@@ -12,6 +12,12 @@ namespace nix {
|
||||
#define WORKER_MAGIC_1 0x6e697863
|
||||
#define WORKER_MAGIC_2 0x6478696f
|
||||
|
||||
/* Note: you generally shouldn't change the protocol version. Define a
|
||||
new `WorkerProto::Feature` instead. */
|
||||
#define PROTOCOL_VERSION (1 << 8 | 39)
|
||||
#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18)
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
#define STDERR_NEXT 0x6f6c6d67
|
||||
#define STDERR_READ 0x64617461 // data needed from source
|
||||
#define STDERR_WRITE 0x64617416 // data for sink
|
||||
@@ -30,6 +36,9 @@ struct BuildResult;
|
||||
struct KeyedBuildResult;
|
||||
struct ValidPathInfo;
|
||||
struct UnkeyedValidPathInfo;
|
||||
struct DrvOutput;
|
||||
struct UnkeyedRealisation;
|
||||
struct Realisation;
|
||||
enum BuildMode : uint8_t;
|
||||
enum TrustedFlag : bool;
|
||||
enum class GCAction;
|
||||
@@ -109,6 +118,12 @@ struct WorkerProto
|
||||
|
||||
static const Version minimum;
|
||||
|
||||
/**
|
||||
* Feature for transmitting `UnkeyedRealisation` and `DrvOutput`
|
||||
* using drvPath (store path) instead of the old hash-based JSON format.
|
||||
*/
|
||||
static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash";
|
||||
|
||||
/**
|
||||
* A unidirectional read connection, to be used by the read half of the
|
||||
* canonical serializers below.
|
||||
@@ -307,6 +322,14 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(DrvOutput);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(UnkeyedRealisation);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(Realisation);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(std::optional<UnkeyedRealisation>);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(BuildMode);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(GCAction);
|
||||
@@ -325,8 +348,8 @@ DECLARE_WORKER_SERIALISER(std::set<T COMMA_ Compare>);
|
||||
template<typename... Ts>
|
||||
DECLARE_WORKER_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
template<typename K, typename V>
|
||||
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V>);
|
||||
template<typename K, typename V, typename Compare>
|
||||
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
|
||||
#undef COMMA_
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -360,14 +360,14 @@ LocalStore::LocalStore(ref<const Config> config)
|
||||
state->stmts->RegisterRealisedOutput.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert into Realisations (drvPath, outputName, outputPath, signatures)
|
||||
insert into BuildTraceV2 (drvPath, outputName, outputPath, signatures)
|
||||
values (?, ?, (select id from ValidPaths where path = ?), ?)
|
||||
;
|
||||
)");
|
||||
state->stmts->UpdateRealisedOutput.create(
|
||||
state->db,
|
||||
R"(
|
||||
update Realisations
|
||||
update BuildTraceV2
|
||||
set signatures = ?
|
||||
where
|
||||
drvPath = ? and
|
||||
@@ -377,16 +377,16 @@ LocalStore::LocalStore(ref<const Config> config)
|
||||
state->stmts->QueryRealisedOutput.create(
|
||||
state->db,
|
||||
R"(
|
||||
select Realisations.id, Output.path, Realisations.signatures from Realisations
|
||||
inner join ValidPaths as Output on Output.id = Realisations.outputPath
|
||||
select BuildTraceV2.id, Output.path, BuildTraceV2.signatures from BuildTraceV2
|
||||
inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath
|
||||
where drvPath = ? and outputName = ?
|
||||
;
|
||||
)");
|
||||
state->stmts->QueryAllRealisedOutputs.create(
|
||||
state->db,
|
||||
R"(
|
||||
select outputName, Output.path from Realisations
|
||||
inner join ValidPaths as Output on Output.id = Realisations.outputPath
|
||||
select outputName, Output.path from BuildTraceV2
|
||||
inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath
|
||||
where drvPath = ?
|
||||
;
|
||||
)");
|
||||
@@ -599,7 +599,7 @@ void LocalStore::upgradeDBSchema(State & state)
|
||||
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
|
||||
doUpgrade(
|
||||
"20220326-ca-derivations",
|
||||
"20251016-ca-derivations",
|
||||
#include "ca-specific-schema.sql.gen.hh"
|
||||
);
|
||||
}
|
||||
@@ -643,8 +643,8 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
auto combinedSignatures = oldR->signatures;
|
||||
combinedSignatures.insert(info.signatures.begin(), info.signatures.end());
|
||||
state->stmts->UpdateRealisedOutput
|
||||
.use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(info.id.strHash())(
|
||||
info.id.outputName)
|
||||
.use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(
|
||||
info.id.drvPath.to_string())(info.id.outputName)
|
||||
.exec();
|
||||
} else {
|
||||
throw Error(
|
||||
@@ -658,7 +658,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
}
|
||||
} else {
|
||||
state->stmts->RegisterRealisedOutput
|
||||
.use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))(
|
||||
.use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))(
|
||||
concatStringsSep(" ", Signature::toStrings(info.signatures)))
|
||||
.exec();
|
||||
}
|
||||
@@ -1534,7 +1534,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const std::set<Signa
|
||||
std::optional<std::pair<int64_t, UnkeyedRealisation>>
|
||||
LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id)
|
||||
{
|
||||
auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName));
|
||||
auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName));
|
||||
if (!useQueryRealisedOutput.next())
|
||||
return std::nullopt;
|
||||
auto realisationDbId = useQueryRealisedOutput.getInt(0);
|
||||
|
||||
@@ -242,16 +242,15 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
|
||||
|
||||
// If there are unknown output paths, attempt to find if the
|
||||
// paths are known to substituters through a realisation.
|
||||
auto outputHashes = staticOutputHashes(*this, *drv);
|
||||
knownOutputPaths = true;
|
||||
|
||||
for (auto [outputName, hash] : outputHashes) {
|
||||
for (auto & [outputName, _] : drv->outputs) {
|
||||
if (!bfd.outputs.contains(outputName))
|
||||
continue;
|
||||
|
||||
bool found = false;
|
||||
for (auto & sub : getDefaultSubstituters()) {
|
||||
auto realisation = sub->queryRealisation({hash, outputName});
|
||||
auto realisation = sub->queryRealisation({drvPath, outputName});
|
||||
if (!realisation)
|
||||
continue;
|
||||
found = true;
|
||||
@@ -368,7 +367,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
|
||||
OutputPathMap outputs;
|
||||
for (auto & [outputName, outputPathOpt] : outputsOpt) {
|
||||
if (!outputPathOpt)
|
||||
throw MissingRealisation(bfd.drvPath->to_string(store), outputName);
|
||||
throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName);
|
||||
auto & outputPath = *outputPathOpt;
|
||||
outputs.insert_or_assign(outputName, outputPath);
|
||||
}
|
||||
@@ -392,7 +391,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
|
||||
bfd.output);
|
||||
auto & optPath = outputPaths.at(bfd.output);
|
||||
if (!optPath)
|
||||
throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output);
|
||||
throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output);
|
||||
return *optPath;
|
||||
},
|
||||
},
|
||||
|
||||
@@ -42,12 +42,18 @@ create table if not exists NARs (
|
||||
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||
);
|
||||
|
||||
create table if not exists Realisations (
|
||||
create table if not exists BuildTrace (
|
||||
cache integer not null,
|
||||
outputId text not null,
|
||||
content blob, -- Json serialisation of the realisation, or null if the realisation is absent
|
||||
|
||||
drvPath text not null,
|
||||
outputName text not null,
|
||||
|
||||
-- The following are null if the realisation is absent
|
||||
outputPath text,
|
||||
sigs text,
|
||||
|
||||
timestamp integer not null,
|
||||
primary key (cache, outputId),
|
||||
primary key (cache, drvPath, outputName),
|
||||
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||
);
|
||||
|
||||
@@ -87,7 +93,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
|
||||
NarInfoDiskCacheImpl(
|
||||
const Settings & settings,
|
||||
SQLiteSettings sqliteSettings,
|
||||
Path dbPath = (getCacheDir() / "binary-cache-v7.sqlite").string())
|
||||
Path dbPath = (getCacheDir() / "binary-cache-v8.sqlite").string())
|
||||
: NarInfoDiskCache{settings}
|
||||
{
|
||||
auto state(_state.lock());
|
||||
@@ -123,24 +129,24 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
|
||||
state->insertRealisation.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert or replace into Realisations(cache, outputId, content, timestamp)
|
||||
values (?, ?, ?, ?)
|
||||
insert or replace into BuildTrace(cache, drvPath, outputName, outputPath, sigs, timestamp)
|
||||
values (?, ?, ?, ?, ?, ?)
|
||||
)");
|
||||
|
||||
state->insertMissingRealisation.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert or replace into Realisations(cache, outputId, timestamp)
|
||||
values (?, ?, ?)
|
||||
insert or replace into BuildTrace(cache, drvPath, outputName, timestamp)
|
||||
values (?, ?, ?, ?)
|
||||
)");
|
||||
|
||||
state->queryRealisation.create(
|
||||
state->db,
|
||||
R"(
|
||||
select content from Realisations
|
||||
where cache = ? and outputId = ? and
|
||||
((content is null and timestamp > ?) or
|
||||
(content is not null and timestamp > ?))
|
||||
select outputPath, sigs from BuildTrace
|
||||
where cache = ? and drvPath = ? and outputName = ? and
|
||||
((outputPath is null and timestamp > ?) or
|
||||
(outputPath is not null and timestamp > ?))
|
||||
)");
|
||||
|
||||
/* Periodically purge expired entries from the database. */
|
||||
@@ -297,22 +303,27 @@ public:
|
||||
|
||||
auto now = time(0);
|
||||
|
||||
auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())(
|
||||
auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(
|
||||
now - settings.ttlNegative)(now - settings.ttlPositive));
|
||||
|
||||
if (!queryRealisation.next())
|
||||
return {oUnknown, 0};
|
||||
return {oUnknown, nullptr};
|
||||
|
||||
if (queryRealisation.isNull(0))
|
||||
return {oInvalid, 0};
|
||||
return {oInvalid, nullptr};
|
||||
|
||||
try {
|
||||
return {
|
||||
oValid,
|
||||
std::make_shared<Realisation>(nlohmann::json::parse(queryRealisation.getStr(0))),
|
||||
std::make_shared<Realisation>(
|
||||
UnkeyedRealisation{
|
||||
.outPath = StorePath{queryRealisation.getStr(0)},
|
||||
.signatures = nlohmann::json::parse(queryRealisation.getStr(1)),
|
||||
},
|
||||
id),
|
||||
};
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing the local disk cache");
|
||||
e.addTrace({}, "reading build trace key-value from the local disk cache");
|
||||
throw;
|
||||
}
|
||||
});
|
||||
@@ -358,7 +369,9 @@ public:
|
||||
auto & cache(getCache(*state, uri));
|
||||
|
||||
state->insertRealisation
|
||||
.use()(cache.id)(realisation.id.to_string())(static_cast<nlohmann::json>(realisation).dump())(time(0))
|
||||
.use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)(
|
||||
realisation.outPath.to_string())(static_cast<nlohmann::json>(realisation.signatures).dump())(
|
||||
time(0))
|
||||
.exec();
|
||||
});
|
||||
}
|
||||
@@ -369,7 +382,7 @@ public:
|
||||
auto state(_state.lock());
|
||||
|
||||
auto & cache(getCache(*state, uri));
|
||||
state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec();
|
||||
state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -8,28 +8,34 @@ namespace nix {
|
||||
|
||||
MakeError(InvalidDerivationOutputId, Error);
|
||||
|
||||
DrvOutput DrvOutput::parse(const std::string & strRep)
|
||||
DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s)
|
||||
{
|
||||
size_t n = strRep.find("!");
|
||||
if (n == strRep.npos)
|
||||
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
|
||||
|
||||
size_t n = s.rfind('^');
|
||||
if (n == s.npos)
|
||||
throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s);
|
||||
return DrvOutput{
|
||||
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
|
||||
.outputName = strRep.substr(n + 1),
|
||||
.drvPath = store.parseStorePath(s.substr(0, n)),
|
||||
.outputName = OutputName{s.substr(n + 1)},
|
||||
};
|
||||
}
|
||||
|
||||
std::string DrvOutput::render(const StoreDirConfig & store) const
|
||||
{
|
||||
return std::string(store.printStorePath(drvPath)) + "^" + outputName;
|
||||
}
|
||||
|
||||
std::string DrvOutput::to_string() const
|
||||
{
|
||||
return strHash() + "!" + outputName;
|
||||
return std::string(drvPath.to_string()) + "^" + outputName;
|
||||
}
|
||||
|
||||
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
|
||||
{
|
||||
nlohmann::json serialized = Realisation{*this, key};
|
||||
serialized.erase("signatures");
|
||||
return serialized.dump();
|
||||
auto serialised = static_cast<nlohmann::json>(Realisation{*this, key});
|
||||
auto value = serialised.find("value");
|
||||
assert(value != serialised.end());
|
||||
value->erase("signatures");
|
||||
return serialised.dump();
|
||||
}
|
||||
|
||||
void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer)
|
||||
@@ -61,9 +67,20 @@ const StorePath & RealisedPath::path() const &
|
||||
return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw);
|
||||
}
|
||||
|
||||
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
|
||||
MissingRealisation::MissingRealisation(
|
||||
const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName)
|
||||
: Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath))
|
||||
{
|
||||
return outPath == other.outPath;
|
||||
}
|
||||
|
||||
MissingRealisation::MissingRealisation(
|
||||
const StoreDirConfig & store,
|
||||
const SingleDerivedPath & drvPath,
|
||||
const StorePath & drvPathResolved,
|
||||
const OutputName & outputName)
|
||||
: MissingRealisation{store, drvPathResolved, outputName}
|
||||
{
|
||||
addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -74,12 +91,20 @@ using namespace nix;
|
||||
|
||||
DrvOutput adl_serializer<DrvOutput>::from_json(const json & json)
|
||||
{
|
||||
return DrvOutput::parse(getString(json));
|
||||
auto obj = getObject(json);
|
||||
|
||||
return {
|
||||
.drvPath = valueAt(obj, "drvPath"),
|
||||
.outputName = getString(valueAt(obj, "outputName")),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<DrvOutput>::to_json(json & json, const DrvOutput & drvOutput)
|
||||
{
|
||||
json = drvOutput.to_string();
|
||||
json = {
|
||||
{"drvPath", drvOutput.drvPath},
|
||||
{"outputName", drvOutput.outputName},
|
||||
};
|
||||
}
|
||||
|
||||
UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & json0)
|
||||
@@ -101,25 +126,25 @@ void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedReali
|
||||
json = {
|
||||
{"outPath", r.outPath},
|
||||
{"signatures", r.signatures},
|
||||
// back-compat
|
||||
{"dependentRealisations", json::object()},
|
||||
};
|
||||
}
|
||||
|
||||
Realisation adl_serializer<Realisation>::from_json(const json & json0)
|
||||
Realisation adl_serializer<Realisation>::from_json(const json & json)
|
||||
{
|
||||
auto json = getObject(json0);
|
||||
auto obj = getObject(json);
|
||||
|
||||
return Realisation{
|
||||
static_cast<UnkeyedRealisation>(json0),
|
||||
valueAt(json, "id"),
|
||||
return {
|
||||
static_cast<UnkeyedRealisation>(valueAt(obj, "value")),
|
||||
static_cast<DrvOutput>(valueAt(obj, "key")),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<Realisation>::to_json(json & json, const Realisation & r)
|
||||
{
|
||||
json = static_cast<const UnkeyedRealisation &>(r);
|
||||
json["id"] = r.id;
|
||||
json = {
|
||||
{"key", r.id},
|
||||
{"value", static_cast<const UnkeyedRealisation &>(r)},
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
|
||||
@@ -503,12 +503,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << WorkerProto::Op::RegisterDrvOutput;
|
||||
if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
WorkerProto::write(*this, *conn, info.id);
|
||||
conn->to << std::string(info.outPath.to_string());
|
||||
} else {
|
||||
WorkerProto::write(*this, *conn, info);
|
||||
}
|
||||
WorkerProto::write(*this, *conn, info);
|
||||
conn.processStderr();
|
||||
}
|
||||
|
||||
@@ -518,8 +513,10 @@ void RemoteStore::queryRealisationUncached(
|
||||
try {
|
||||
auto conn(getConnection());
|
||||
|
||||
if (conn->protoVersion.number < WorkerProto::Version::Number{1, 27}) {
|
||||
warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4");
|
||||
if (!conn->protoVersion.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
warn(
|
||||
"the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
return callback(nullptr);
|
||||
}
|
||||
|
||||
@@ -527,21 +524,12 @@ void RemoteStore::queryRealisationUncached(
|
||||
WorkerProto::write(*this, *conn, id);
|
||||
conn.processStderr();
|
||||
|
||||
auto real = [&]() -> std::shared_ptr<const UnkeyedRealisation> {
|
||||
if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read(*this, *conn);
|
||||
if (outPaths.empty())
|
||||
return nullptr;
|
||||
return std::make_shared<const UnkeyedRealisation>(UnkeyedRealisation{.outPath = *outPaths.begin()});
|
||||
} else {
|
||||
auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read(*this, *conn);
|
||||
if (realisations.empty())
|
||||
return nullptr;
|
||||
return std::make_shared<const UnkeyedRealisation>(*realisations.begin());
|
||||
}
|
||||
}();
|
||||
|
||||
callback(std::shared_ptr<const UnkeyedRealisation>(real));
|
||||
callback([&]() -> std::shared_ptr<const UnkeyedRealisation> {
|
||||
auto realisation = WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(*this, *conn);
|
||||
if (!realisation)
|
||||
return nullptr;
|
||||
return std::make_shared<const UnkeyedRealisation>(*realisation);
|
||||
}());
|
||||
} catch (...) {
|
||||
return callback.rethrow();
|
||||
}
|
||||
@@ -623,30 +611,19 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
|
||||
|
||||
OutputPathMap outputs;
|
||||
auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath);
|
||||
auto drv = evalStore->readDerivation(drvPath);
|
||||
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
|
||||
auto built = resolveDerivedPath(*this, bfd, &*evalStore);
|
||||
for (auto & [output, outputPath] : built) {
|
||||
auto outputHash = get(outputHashes, output);
|
||||
if (!outputHash)
|
||||
throw Error(
|
||||
"the derivation '%s' doesn't have an output named '%s'",
|
||||
printStorePath(drvPath),
|
||||
output);
|
||||
auto outputId = DrvOutput{*outputHash, output};
|
||||
auto outputId = DrvOutput{drvPath, output};
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
auto realisation = queryRealisation(outputId);
|
||||
if (!realisation)
|
||||
throw MissingRealisation(outputId);
|
||||
success.builtOutputs.emplace(output, Realisation{*realisation, outputId});
|
||||
throw MissingRealisation(*this, outputId);
|
||||
success.builtOutputs.emplace(output, *realisation);
|
||||
} else {
|
||||
success.builtOutputs.emplace(
|
||||
output,
|
||||
Realisation{
|
||||
UnkeyedRealisation{
|
||||
.outPath = outputPath,
|
||||
},
|
||||
outputId,
|
||||
UnkeyedRealisation{
|
||||
.outPath = outputPath,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,9 +280,18 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
|
||||
|
||||
for (auto & result : results) {
|
||||
if (auto * successP = result.tryGetSuccess()) {
|
||||
for (auto & [outputName, output] : successP->builtOutputs) {
|
||||
newPaths.insert(output.outPath);
|
||||
newRealisations.insert(output);
|
||||
if (auto * pathBuilt = std::get_if<DerivedPathBuilt>(&result.path)) {
|
||||
// TODO ugly extra IO
|
||||
auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath);
|
||||
for (auto & [outputName, output] : successP->builtOutputs) {
|
||||
newPaths.insert(output.outPath);
|
||||
newRealisations.insert(
|
||||
{output,
|
||||
{
|
||||
.drvPath = drvPath,
|
||||
.outputName = outputName,
|
||||
}});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include "nix/store/serve-protocol-impl.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/store/path-info.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@@ -28,10 +29,19 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & stor
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 3})
|
||||
conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime;
|
||||
if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(store, conn);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation));
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 8}) {
|
||||
success.builtOutputs = ServeProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
|
||||
} else if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
for (auto & [output, realisation] : ServeProto::Serialise<StringMap>::read(store, conn)) {
|
||||
size_t n = output.find("!");
|
||||
if (n == output.npos)
|
||||
throw Error("Invalid derivation output id %s", output);
|
||||
success.builtOutputs.insert_or_assign(
|
||||
output.substr(n + 1),
|
||||
UnkeyedRealisation{
|
||||
StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}});
|
||||
}
|
||||
}
|
||||
|
||||
res.inner = std::visit(
|
||||
@@ -63,15 +73,18 @@ void ServeProto::Serialise<BuildResult>::write(
|
||||
default value for the fields that don't exist in that case. */
|
||||
auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) {
|
||||
conn.to << errorMsg;
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 3})
|
||||
conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime;
|
||||
if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
DrvOutputs builtOutputsFullKey;
|
||||
for (auto & [output, realisation] : builtOutputs)
|
||||
builtOutputsFullKey.insert_or_assign(realisation.id, realisation);
|
||||
ServeProto::write(store, conn, builtOutputsFullKey);
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 8}) {
|
||||
ServeProto::write(store, conn, builtOutputs);
|
||||
} else if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
// We no longer support these types of realisations
|
||||
ServeProto::write(store, conn, StringMap{});
|
||||
}
|
||||
};
|
||||
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const BuildResult::Failure & failure) {
|
||||
@@ -158,4 +171,82 @@ void ServeProto::Serialise<ServeProto::BuildOptions>::write(
|
||||
}
|
||||
}
|
||||
|
||||
UnkeyedRealisation ServeProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
|
||||
auto outPath = ServeProto::Serialise<StorePath>::read(store, conn);
|
||||
auto signatures = ServeProto::Serialise<std::set<Signature>>::read(store, conn);
|
||||
|
||||
return UnkeyedRealisation{
|
||||
.outPath = std::move(outPath),
|
||||
.signatures = std::move(signatures),
|
||||
};
|
||||
}
|
||||
|
||||
void ServeProto::Serialise<UnkeyedRealisation>::write(
|
||||
const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
ServeProto::write(store, conn, info.outPath);
|
||||
ServeProto::write(store, conn, info.signatures);
|
||||
}
|
||||
|
||||
DrvOutput ServeProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
|
||||
auto drvPath = ServeProto::Serialise<StorePath>::read(store, conn);
|
||||
auto outputName = ServeProto::Serialise<std::string>::read(store, conn);
|
||||
|
||||
return DrvOutput{
|
||||
.drvPath = std::move(drvPath),
|
||||
.outputName = std::move(outputName),
|
||||
};
|
||||
}
|
||||
|
||||
void ServeProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
ServeProto::write(store, conn, info.drvPath);
|
||||
ServeProto::write(store, conn, info.outputName);
|
||||
}
|
||||
|
||||
Realisation ServeProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
auto id = ServeProto::Serialise<DrvOutput>::read(store, conn);
|
||||
auto unkeyed = ServeProto::Serialise<UnkeyedRealisation>::read(store, conn);
|
||||
|
||||
return Realisation{
|
||||
std::move(unkeyed),
|
||||
std::move(id),
|
||||
};
|
||||
}
|
||||
|
||||
void ServeProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
|
||||
{
|
||||
ServeProto::write(store, conn, info.id);
|
||||
ServeProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -372,9 +372,8 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore
|
||||
return outputs;
|
||||
|
||||
auto drv = evalStore.readInvalidDerivation(path);
|
||||
auto drvHashes = staticOutputHashes(*this, drv);
|
||||
for (auto & [outputName, hash] : drvHashes) {
|
||||
auto realisation = queryRealisation(DrvOutput{hash, outputName});
|
||||
for (auto & [outputName, _] : drv.outputs) {
|
||||
auto realisation = queryRealisation(DrvOutput{path, outputName});
|
||||
if (realisation) {
|
||||
outputs.insert_or_assign(outputName, realisation->outPath);
|
||||
} else {
|
||||
@@ -394,7 +393,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev
|
||||
OutputPathMap result;
|
||||
for (auto & [outName, optOutPath] : resp) {
|
||||
if (!optOutPath)
|
||||
throw MissingRealisation(printStorePath(path), outName);
|
||||
throw MissingRealisation(*this, path, outName);
|
||||
result.insert_or_assign(outName, *optOutPath);
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -1958,7 +1958,10 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
|
||||
{
|
||||
.outPath = newInfo.path,
|
||||
},
|
||||
DrvOutput{oldinfo->outputHash, outputName},
|
||||
DrvOutput{
|
||||
.drvPath = drvPath,
|
||||
.outputName = outputName,
|
||||
},
|
||||
};
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) {
|
||||
store.signRealisation(thisRealisation);
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "nix/store/worker-protocol-impl.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/store/path-info.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -20,6 +21,12 @@ const WorkerProto::Version WorkerProto::latest = {
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features =
|
||||
{
|
||||
std::string{
|
||||
WorkerProto::featureRealisationWithPath,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const WorkerProto::Version WorkerProto::minimum = {
|
||||
@@ -254,10 +261,19 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & sto
|
||||
res.cpuUser = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
|
||||
res.cpuSystem = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
|
||||
}
|
||||
if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation));
|
||||
|
||||
if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
success.builtOutputs = WorkerProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
|
||||
} else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
for (auto && [output, realisation] : WorkerProto::Serialise<StringMap>::read(store, conn)) {
|
||||
size_t n = output.find("!");
|
||||
if (n == output.npos)
|
||||
throw Error("Invalid derivation output id %s", output);
|
||||
success.builtOutputs.insert_or_assign(
|
||||
output.substr(n + 1),
|
||||
UnkeyedRealisation{
|
||||
StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}});
|
||||
}
|
||||
}
|
||||
|
||||
res.inner = std::visit(
|
||||
@@ -296,13 +312,15 @@ void WorkerProto::Serialise<BuildResult>::write(
|
||||
WorkerProto::write(store, conn, res.cpuUser);
|
||||
WorkerProto::write(store, conn, res.cpuSystem);
|
||||
}
|
||||
if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
DrvOutputs builtOutputsFullKey;
|
||||
for (auto & [output, realisation] : builtOutputs)
|
||||
builtOutputsFullKey.insert_or_assign(realisation.id, realisation);
|
||||
WorkerProto::write(store, conn, builtOutputsFullKey);
|
||||
|
||||
if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
WorkerProto::write(store, conn, builtOutputs);
|
||||
} else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
// Don't support those types of realisations anymore.
|
||||
WorkerProto::write(store, conn, StringMap{});
|
||||
}
|
||||
};
|
||||
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const BuildResult::Failure & failure) {
|
||||
@@ -395,4 +413,109 @@ void WorkerProto::Serialise<WorkerProto::ClientHandshakeInfo>::write(
|
||||
}
|
||||
}
|
||||
|
||||
UnkeyedRealisation WorkerProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to understand build trace",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
|
||||
auto outPath = WorkerProto::Serialise<StorePath>::read(store, conn);
|
||||
auto signatures = WorkerProto::Serialise<std::set<Signature>>::read(store, conn);
|
||||
|
||||
return UnkeyedRealisation{
|
||||
.outPath = std::move(outPath),
|
||||
.signatures = std::move(signatures),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<UnkeyedRealisation>::write(
|
||||
const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to understand build trace",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
WorkerProto::write(store, conn, info.outPath);
|
||||
WorkerProto::write(store, conn, info.signatures);
|
||||
}
|
||||
|
||||
std::optional<UnkeyedRealisation>
|
||||
WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
// Hack to improve compat
|
||||
(void) WorkerProto::Serialise<std::string>::read(store, conn);
|
||||
return std::nullopt;
|
||||
} else {
|
||||
auto temp = readNum<uint8_t>(conn.from);
|
||||
switch (temp) {
|
||||
case 0:
|
||||
return std::nullopt;
|
||||
case 1:
|
||||
return WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
|
||||
default:
|
||||
throw Error("Invalid optional build trace from remote");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::write(
|
||||
const StoreDirConfig & store, WriteConn conn, const std::optional<UnkeyedRealisation> & info)
|
||||
{
|
||||
if (!info) {
|
||||
conn.to << uint8_t{0};
|
||||
} else {
|
||||
conn.to << uint8_t{1};
|
||||
WorkerProto::write(store, conn, *info);
|
||||
}
|
||||
}
|
||||
|
||||
DrvOutput WorkerProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
|
||||
auto drvPath = WorkerProto::Serialise<StorePath>::read(store, conn);
|
||||
auto outputName = WorkerProto::Serialise<std::string>::read(store, conn);
|
||||
|
||||
return DrvOutput{
|
||||
.drvPath = std::move(drvPath),
|
||||
.outputName = std::move(outputName),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
WorkerProto::write(store, conn, info.drvPath);
|
||||
WorkerProto::write(store, conn, info.outputName);
|
||||
}
|
||||
|
||||
Realisation WorkerProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
auto id = WorkerProto::Serialise<DrvOutput>::read(store, conn);
|
||||
auto unkeyed = WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
|
||||
|
||||
return Realisation{
|
||||
std::move(unkeyed),
|
||||
std::move(id),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
|
||||
{
|
||||
WorkerProto::write(store, conn, info.id);
|
||||
WorkerProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -346,13 +346,11 @@ static int main_build_remote(int argc, char ** argv)
|
||||
optResult = std::move(res[0]);
|
||||
}
|
||||
|
||||
auto outputHashes = staticOutputHashes(*store, drv);
|
||||
std::set<Realisation> missingRealisations;
|
||||
StorePathSet missingPaths;
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
|
||||
for (auto & outputName : wantedOutputs) {
|
||||
auto thisOutputHash = outputHashes.at(outputName);
|
||||
auto thisOutputId = DrvOutput{thisOutputHash, outputName};
|
||||
auto thisOutputId = DrvOutput{*drvPath, outputName};
|
||||
if (!store->queryRealisation(thisOutputId)) {
|
||||
debug("missing output %s", outputName);
|
||||
assert(optResult);
|
||||
@@ -362,7 +360,7 @@ static int main_build_remote(int argc, char ** argv)
|
||||
auto i = success.builtOutputs.find(outputName);
|
||||
assert(i != success.builtOutputs.end());
|
||||
auto & newRealisation = i->second;
|
||||
missingRealisations.insert(newRealisation);
|
||||
missingRealisations.insert({newRealisation, thisOutputId});
|
||||
missingPaths.insert(newRealisation.outPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,10 +163,13 @@ StoreWrapper::queryPathInfo(char * path, int base32)
|
||||
}
|
||||
|
||||
SV *
|
||||
StoreWrapper::queryRawRealisation(char * outputId)
|
||||
StoreWrapper::queryRawRealisation(char * drvPath, char * outputName)
|
||||
PPCODE:
|
||||
try {
|
||||
auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId));
|
||||
auto realisation = THIS->store->queryRealisation(DrvOutput{
|
||||
.drvPath = THIS->store->parseStorePath(drvPath),
|
||||
.outputName = outputName,
|
||||
});
|
||||
if (realisation)
|
||||
XPUSHs(sv_2mortal(newSVpv(static_cast<nlohmann::json>(*realisation).dump().c_str(), 0)));
|
||||
else
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# shellcheck shell=bash
|
||||
source ../common.sh
|
||||
|
||||
# Need backend to support revamped CA
|
||||
requireDaemonNewerThan "2.34.0pre20251217"
|
||||
|
||||
enableFeatures "ca-derivations"
|
||||
|
||||
TODO_NixOS
|
||||
|
||||
@@ -49,14 +49,14 @@ fi
|
||||
clearStore
|
||||
nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore"
|
||||
clearStore
|
||||
rm -r "$REMOTE_STORE_DIR/realisations"
|
||||
rm -r "$REMOTE_STORE_DIR/build-trace-v2"
|
||||
nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
|
||||
# There's no easy way to check whether a realisation is present on the local
|
||||
# store − short of manually querying the db, but the build environment doesn't
|
||||
# have the sqlite binary − so we instead push things again, and check that the
|
||||
# realisations have correctly been pushed to the remote store
|
||||
nix copy --to "$REMOTE_STORE" --file ../simple.nix
|
||||
if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
|
||||
if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace-v2")" ]]; then
|
||||
echo "Realisations not rebuilt"
|
||||
exit 1
|
||||
fi
|
||||
@@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
|
||||
# Try rebuilding, but remove the realisations from the remote cache to force
|
||||
# using the cachecache
|
||||
clearStore
|
||||
rm "$REMOTE_STORE_DIR"/realisations/*
|
||||
rm -r "$REMOTE_STORE_DIR"/build-trace-v2/*
|
||||
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
|
||||
|
||||
Reference in New Issue
Block a user