Reapply "Use the hash modulo in the derivation outputs"
This reverts commit 4f91e9599f.
This broke:
when I applied this pr, `--print-out-paths` wouldn't print anything:
```
shell-for-nix-env % nix build --print-out-paths .#legacyPackages.aarch64-darwin.homeConfigurations.macos.activationPackage
```
After dropping the patches from my fork, it does again.
```
shell-for-nix-env % nix build --print-out-paths .#legacyPackages.aarch64-darwin.homeConfigurations.macos.activationPackage
/nix/store/s8mlcalszdml0v8172w4hwqnx0m6477r-home-manager-generation
```
This commit is contained in:
@@ -83,7 +83,7 @@ properties:
|
||||
description: |
|
||||
A mapping from output names to their build trace entries.
|
||||
additionalProperties:
|
||||
"$ref": "build-trace-entry-v2.yaml#/$defs/value"
|
||||
"$ref": "build-trace-entry-v2.yaml"
|
||||
|
||||
failure:
|
||||
type: object
|
||||
|
||||
@@ -16,21 +16,24 @@ description: |
|
||||
|
||||
- Version 1: Original format
|
||||
|
||||
- Version 2:
|
||||
- Use `drvPath` not `drvHash` to refer to derivation in a more conventional way.
|
||||
- Remove `dependentRealisations`
|
||||
- Separate into `key` and `value`
|
||||
- Version 2: Remove `dependentRealisations`
|
||||
|
||||
type: object
|
||||
required:
|
||||
- key
|
||||
- value
|
||||
- id
|
||||
- outPath
|
||||
- signatures
|
||||
allOf:
|
||||
- "$ref": "#/$defs/key"
|
||||
- "$ref": "#/$defs/value"
|
||||
properties:
|
||||
key:
|
||||
"$ref": "#/$defs/key"
|
||||
value:
|
||||
"$ref": "#/$defs/value"
|
||||
additionalProperties: false
|
||||
id: {}
|
||||
outPath: {}
|
||||
signatures: {}
|
||||
additionalProperties:
|
||||
dependentRealisations:
|
||||
description: deprecated field
|
||||
type: object
|
||||
|
||||
"$defs":
|
||||
key:
|
||||
@@ -40,20 +43,23 @@ additionalProperties: false
|
||||
This is the "key" part, refering to a derivation and output.
|
||||
type: object
|
||||
required:
|
||||
- drvPath
|
||||
- outputName
|
||||
- id
|
||||
properties:
|
||||
drvPath:
|
||||
"$ref": "store-path-v1.yaml"
|
||||
title: Derivation Path
|
||||
description: |
|
||||
The store path of the derivation that was built.
|
||||
outputName:
|
||||
id:
|
||||
type: string
|
||||
title: Output Name
|
||||
title: Derivation Output ID
|
||||
pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$"
|
||||
description: |
|
||||
Name of the specific output (e.g., "out", "dev", "doc")
|
||||
additionalProperties: false
|
||||
Unique identifier for the derivation output that was built.
|
||||
|
||||
Format: `{hash-quotient-drv}!{output-name}`
|
||||
|
||||
- **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv).
|
||||
Begins with `sha256:`.
|
||||
|
||||
- **output-name**: Name of the specific output (e.g., "out", "dev", "doc")
|
||||
|
||||
Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
|
||||
|
||||
value:
|
||||
title: Build Trace Value
|
||||
@@ -71,6 +77,13 @@ additionalProperties: false
|
||||
description: |
|
||||
The path to the store object that resulted from building this derivation for the given output name.
|
||||
|
||||
patternProperties:
|
||||
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
|
||||
"$ref": "store-path-v1.yaml"
|
||||
title: Dependent Store Path
|
||||
description: Store path that this dependency resolved to during the build
|
||||
additionalProperties: false
|
||||
|
||||
signatures:
|
||||
type: array
|
||||
title: Build Signatures
|
||||
|
||||
@@ -65,6 +65,9 @@ schemas = [
|
||||
'schema' : schema_dir / 'build-trace-entry-v2.yaml',
|
||||
'files' : [
|
||||
'simple.json',
|
||||
# The field is no longer supported, but we want to show that we
|
||||
# ignore it during parsing.
|
||||
'with-dependent-realisations.json',
|
||||
'with-signature.json',
|
||||
],
|
||||
},
|
||||
|
||||
@@ -108,16 +108,20 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
|
||||
overloaded{
|
||||
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
|
||||
[&](const BuiltPath::Built & p) {
|
||||
auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
|
||||
for (auto & [outputName, outputPath] : p.outputs) {
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
DrvOutput key{
|
||||
.drvPath = p.drvPath->outPath(),
|
||||
.outputName = outputName,
|
||||
};
|
||||
auto drvOutput = get(drvHashes, outputName);
|
||||
if (!drvOutput)
|
||||
throw Error(
|
||||
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
|
||||
store.printStorePath(p.drvPath->outPath()),
|
||||
outputName);
|
||||
DrvOutput key{*drvOutput, outputName};
|
||||
auto thisRealisation = store.queryRealisation(key);
|
||||
// We’ve built it, so we must have the realisation.
|
||||
assert(thisRealisation);
|
||||
res.insert(Realisation{*thisRealisation, key});
|
||||
assert(thisRealisation); // We’ve built it, so we must
|
||||
// have the realisation
|
||||
res.insert(Realisation{*thisRealisation, std::move(key)});
|
||||
} else {
|
||||
res.insert(outputPath);
|
||||
}
|
||||
|
||||
@@ -75,13 +75,25 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -108,6 +108,69 @@ CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
(std::tuple<DrvOutput, DrvOutput>{
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
realisation,
|
||||
"realisation",
|
||||
(std::tuple<Realisation, Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
READ_CHARACTERIZATION_TEST(
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
(std::tuple<Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
vector,
|
||||
"vector",
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
{
|
||||
"builtOutputs": {
|
||||
"bar": {
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
|
||||
"signatures": []
|
||||
},
|
||||
"foo": {
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv": {
|
||||
"ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
{
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
|
||||
"outputName": "foo"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"dependentRealisations": {
|
||||
"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"
|
||||
},
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": []
|
||||
}
|
||||
@@ -1,12 +1,8 @@
|
||||
{
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
|
||||
"outputName": "foo"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
"dependentRealisations": {},
|
||||
"id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo",
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -1,37 +0,0 @@
|
||||
[
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": false,
|
||||
"startTime": 0,
|
||||
"status": "OutputRejected",
|
||||
"stopTime": 0,
|
||||
"success": false,
|
||||
"timesBuilt": 0
|
||||
},
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": true,
|
||||
"startTime": 30,
|
||||
"status": "NotDeterministic",
|
||||
"stopTime": 50,
|
||||
"success": false,
|
||||
"timesBuilt": 3
|
||||
},
|
||||
{
|
||||
"builtOutputs": {
|
||||
"bar": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
|
||||
"signatures": []
|
||||
},
|
||||
"foo": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"startTime": 30,
|
||||
"status": "Built",
|
||||
"stopTime": 50,
|
||||
"success": true,
|
||||
"timesBuilt": 1
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -1,10 +0,0 @@
|
||||
[
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "quux"
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@@ -1,39 +0,0 @@
|
||||
[
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": false,
|
||||
"startTime": 0,
|
||||
"status": "OutputRejected",
|
||||
"stopTime": 0,
|
||||
"success": false,
|
||||
"timesBuilt": 0
|
||||
},
|
||||
{
|
||||
"errorMsg": "no idea why",
|
||||
"isNonDeterministic": true,
|
||||
"startTime": 30,
|
||||
"status": "NotDeterministic",
|
||||
"stopTime": 50,
|
||||
"success": false,
|
||||
"timesBuilt": 3
|
||||
},
|
||||
{
|
||||
"builtOutputs": {
|
||||
"bar": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
|
||||
"signatures": []
|
||||
},
|
||||
"foo": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"cpuSystem": 604000000,
|
||||
"cpuUser": 500000000,
|
||||
"startTime": 30,
|
||||
"status": "Built",
|
||||
"stopTime": 50,
|
||||
"success": true,
|
||||
"timesBuilt": 1
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -1,10 +0,0 @@
|
||||
[
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
{
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "quux"
|
||||
}
|
||||
]
|
||||
Binary file not shown.
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"key": {
|
||||
"drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
|
||||
"outputName": "baz"
|
||||
},
|
||||
"value": {
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
|
||||
"signatures": [
|
||||
"asdf:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"qwer:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
|
||||
]
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
|
||||
@@ -37,19 +37,20 @@ TEST(DummyStore, realisation_read)
|
||||
return cfg->openDummyStore();
|
||||
}();
|
||||
|
||||
StorePath drvPath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"};
|
||||
auto drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16);
|
||||
|
||||
auto outputName = "foo";
|
||||
|
||||
EXPECT_EQ(store->queryRealisation({drvPath, outputName}), nullptr);
|
||||
EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr);
|
||||
|
||||
UnkeyedRealisation value{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
};
|
||||
|
||||
store->buildTrace.insert({drvPath, {{outputName, value}}});
|
||||
store->buildTrace.insert({drvHash, {{outputName, value}}});
|
||||
|
||||
auto value2 = store->queryRealisation({drvPath, outputName});
|
||||
auto value2 = store->queryRealisation({drvHash, outputName});
|
||||
|
||||
ASSERT_TRUE(value2);
|
||||
EXPECT_EQ(*value2, value);
|
||||
@@ -130,7 +131,10 @@ INSTANTIATE_TEST_SUITE_P(DummyStoreJSON, DummyStoreJsonTest, [] {
|
||||
[&] {
|
||||
auto store = writeCfg->openDummyStore();
|
||||
store->buildTrace.insert_or_assign(
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"},
|
||||
Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
|
||||
@@ -46,10 +46,13 @@ TEST_P(RealisationJsonTest, to_json)
|
||||
|
||||
Realisation simple{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
},
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv"},
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
};
|
||||
@@ -74,4 +77,14 @@ INSTANTIATE_TEST_SUITE_P(
|
||||
}(),
|
||||
}));
|
||||
|
||||
/**
|
||||
* We no longer have a notion of "dependent realisations", but we still
|
||||
* want to parse old realisation files. So make this just be a read test
|
||||
* (no write direction), accordingly.
|
||||
*/
|
||||
TEST_F(RealisationTest, dependent_realisations_from_json)
|
||||
{
|
||||
readJsonTest("with-dependent-realisations", simple);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -78,19 +78,16 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
drvOutput_2_8,
|
||||
"drv-output-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
defaultVersion,
|
||||
(std::tuple<DrvOutput, DrvOutput>{
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
@@ -99,37 +96,54 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
unkeyedRealisation_2_8,
|
||||
"unkeyed-realisation-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
(UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
realisation,
|
||||
"realisation",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation, Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
realisation_2_8,
|
||||
"realisation-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
(Realisation{
|
||||
UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
},
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "baz",
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
@@ -195,10 +209,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
/* We now do a lossy read which does not allow us to faithfully write
|
||||
back, since we changed the data type. We still however want to test
|
||||
that this read works, and so for that we have a one-way test. */
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
buildResult_2_6,
|
||||
"build-result-2.6",
|
||||
@@ -231,72 +242,27 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
.timesBuilt = 1,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
#if 0
|
||||
// These fields are not yet serialized.
|
||||
// FIXME Include in next version of protocol or document
|
||||
// why they are skipped.
|
||||
.cpuUser = std::chrono::milliseconds(500s),
|
||||
.cpuSystem = std::chrono::milliseconds(604s),
|
||||
#endif
|
||||
},
|
||||
};
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
buildResult_2_8,
|
||||
"build-result-2.8",
|
||||
(ServeProto::Version{
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
}),
|
||||
({
|
||||
using namespace std::literals::chrono_literals;
|
||||
std::tuple<BuildResult, BuildResult, BuildResult> t{
|
||||
BuildResult{.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::OutputRejected,
|
||||
.msg = HintFmt("no idea why"),
|
||||
}}}},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::NotDeterministic,
|
||||
.msg = HintFmt("no idea why"),
|
||||
.isNonDeterministic = true,
|
||||
}}},
|
||||
.timesBuilt = 3,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Success{
|
||||
.status = BuildResult::Success::Built,
|
||||
.builtOutputs =
|
||||
{
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -18,17 +18,17 @@ namespace nix {
|
||||
TEST(WorkerProtoVersionNumber, ordering)
|
||||
{
|
||||
using Number = WorkerProto::Version::Number;
|
||||
EXPECT_LT((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 20}));
|
||||
EXPECT_GT((Number{.major = 1, .minor = 30}), (Number{.major = 1, .minor = 20}));
|
||||
EXPECT_EQ((Number{.major = 1, .minor = 10}), (Number{.major = 1, .minor = 10}));
|
||||
EXPECT_LT((Number{.major = 0, .minor = 255}), (Number{.major = 1, .minor = 0}));
|
||||
EXPECT_LT((Number{1, 10}), (Number{1, 20}));
|
||||
EXPECT_GT((Number{1, 30}), (Number{1, 20}));
|
||||
EXPECT_EQ((Number{1, 10}), (Number{1, 10}));
|
||||
EXPECT_LT((Number{0, 255}), (Number{1, 0}));
|
||||
}
|
||||
|
||||
TEST(WorkerProtoVersion, partialOrderingSameFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}};
|
||||
V v2{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}};
|
||||
V v1{.number = {1, 20}, .features = {"a", "b"}};
|
||||
V v2{.number = {1, 30}, .features = {"a", "b"}};
|
||||
|
||||
EXPECT_TRUE(v1 < v2);
|
||||
EXPECT_TRUE(v2 > v1);
|
||||
@@ -40,8 +40,8 @@ TEST(WorkerProtoVersion, partialOrderingSameFeatures)
|
||||
TEST(WorkerProtoVersion, partialOrderingSubsetFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
V fewer{.number = {.major = 1, .minor = 30}, .features = {"a"}};
|
||||
V more{.number = {.major = 1, .minor = 30}, .features = {"a", "b"}};
|
||||
V fewer{.number = {1, 30}, .features = {"a"}};
|
||||
V more{.number = {1, 30}, .features = {"a", "b"}};
|
||||
|
||||
// fewer <= more: JUST the features are a subset
|
||||
EXPECT_TRUE(fewer < more);
|
||||
@@ -54,8 +54,8 @@ TEST(WorkerProtoVersion, partialOrderingUnordered)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
// Same number but incomparable features
|
||||
V v1{.number = {.major = 1, .minor = 20}, .features = {"a", "c"}};
|
||||
V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}};
|
||||
V v1{.number = {1, 20}, .features = {"a", "c"}};
|
||||
V v2{.number = {1, 20}, .features = {"a", "b"}};
|
||||
|
||||
EXPECT_FALSE(v1 < v2);
|
||||
EXPECT_FALSE(v1 > v2);
|
||||
@@ -69,8 +69,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
// Higher number but fewer features — unordered
|
||||
V v1{.number = {.major = 1, .minor = 30}, .features = {"a"}};
|
||||
V v2{.number = {.major = 1, .minor = 20}, .features = {"a", "b"}};
|
||||
V v1{.number = {1, 30}, .features = {"a"}};
|
||||
V v2{.number = {1, 20}, .features = {"a", "b"}};
|
||||
|
||||
EXPECT_FALSE(v1 < v2);
|
||||
EXPECT_FALSE(v1 > v2);
|
||||
@@ -80,8 +80,8 @@ TEST(WorkerProtoVersion, partialOrderingHigherNumberFewerFeatures)
|
||||
TEST(WorkerProtoVersion, partialOrderingEmptyFeatures)
|
||||
{
|
||||
using V = WorkerProto::Version;
|
||||
V empty{.number = {.major = 1, .minor = 20}, .features = {}};
|
||||
V some{.number = {.major = 1, .minor = 30}, .features = {"a"}};
|
||||
V empty{.number = {1, 20}, .features = {}};
|
||||
V some{.number = {1, 30}, .features = {"a"}};
|
||||
|
||||
// empty features is a subset of everything
|
||||
EXPECT_TRUE(empty < some);
|
||||
@@ -223,67 +223,69 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
drvOutput,
|
||||
"drv-output-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
"drv-output",
|
||||
defaultVersion,
|
||||
(std::tuple<DrvOutput, DrvOutput>{
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
unkeyedRealisation_realisation_with_path,
|
||||
"unkeyed-realisation-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
realisation,
|
||||
"realisation",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation, Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
(UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
Realisation{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
realisation_realisation_with_path,
|
||||
"realisation-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
defaultVersion,
|
||||
(std::tuple<Realisation>{
|
||||
Realisation{
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{
|
||||
Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
(Realisation{
|
||||
UnkeyedRealisation{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures =
|
||||
{Signature{.keyName = "asdf", .sig = std::string(64, '\0')},
|
||||
Signature{.keyName = "qwer", .sig = std::string(64, '\0')}},
|
||||
},
|
||||
{
|
||||
.drvPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
.outputName = "baz",
|
||||
},
|
||||
}))
|
||||
|
||||
@@ -316,10 +318,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
/* We now do a lossy read which does not allow us to faithfully write
|
||||
back, since we changed the data type. We still however want to test
|
||||
that this read works, and so for that we have a one-way test. */
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_1_28,
|
||||
"build-result-1.28",
|
||||
@@ -348,13 +347,25 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -363,8 +374,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
// See above note
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_1_29,
|
||||
"build-result-1.29",
|
||||
@@ -400,13 +410,27 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -419,8 +443,7 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
t;
|
||||
}))
|
||||
|
||||
// See above note
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_1_37,
|
||||
"build-result-1.37",
|
||||
@@ -456,71 +479,27 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "foo",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
.timesBuilt = 1,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
.cpuUser = std::chrono::microseconds(500s),
|
||||
.cpuSystem = std::chrono::microseconds(604s),
|
||||
},
|
||||
};
|
||||
t;
|
||||
}))
|
||||
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult_realisation_with_path,
|
||||
"build-result-realisation-with-path-not-hash",
|
||||
(WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features = {"realisation-with-path-not-hash"},
|
||||
}),
|
||||
({
|
||||
using namespace std::literals::chrono_literals;
|
||||
std::tuple<BuildResult, BuildResult, BuildResult> t{
|
||||
BuildResult{.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::OutputRejected,
|
||||
.msg = HintFmt("no idea why"),
|
||||
}}}},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Failure{{
|
||||
.status = BuildResult::Failure::NotDeterministic,
|
||||
.msg = HintFmt("no idea why"),
|
||||
.isNonDeterministic = true,
|
||||
}}},
|
||||
.timesBuilt = 3,
|
||||
.startTime = 30,
|
||||
.stopTime = 50,
|
||||
},
|
||||
BuildResult{
|
||||
.inner{BuildResult::Success{
|
||||
.status = BuildResult::Success::Built,
|
||||
.builtOutputs =
|
||||
{
|
||||
{
|
||||
"foo",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"bar",
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"},
|
||||
},
|
||||
DrvOutput{
|
||||
.drvHash =
|
||||
Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "bar",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -941,11 +920,7 @@ TEST_F(WorkerProtoTest, handshake_features)
|
||||
out,
|
||||
in,
|
||||
WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 123,
|
||||
},
|
||||
.number = {.major = 1, .minor = 123},
|
||||
.features = {"bar", "aap", "mies", "xyzzy"},
|
||||
});
|
||||
});
|
||||
@@ -956,11 +931,7 @@ TEST_F(WorkerProtoTest, handshake_features)
|
||||
out,
|
||||
in,
|
||||
WorkerProto::Version{
|
||||
.number =
|
||||
{
|
||||
.major = 1,
|
||||
.minor = 200,
|
||||
},
|
||||
.number = {.major = 1, .minor = 200},
|
||||
.features = {"foo", "bar", "xyzzy"},
|
||||
});
|
||||
|
||||
|
||||
@@ -198,6 +198,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("ca-drv/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo of the derivation
|
||||
// For CA floating derivations, the kind is Deferred since outputs aren't known until build
|
||||
auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
|
||||
ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto drvHash = hashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object
|
||||
auto outputPath = substituter->addToStore(
|
||||
"test-ca-drv-out",
|
||||
@@ -216,7 +222,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
|
||||
// Add the realisation (build trace) to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
drvPath,
|
||||
drvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
@@ -230,7 +236,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
checkpointJson("ca-drv/substituter", substituter);
|
||||
|
||||
// The realisation should not exist in the destination store yet
|
||||
DrvOutput drvOutput{drvPath, "out"};
|
||||
DrvOutput drvOutput{drvHash, "out"};
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
@@ -293,6 +299,11 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
// Write the dependency derivation to the destination store
|
||||
auto depDrvPath = dummyStore->writeDerivation(depDrv);
|
||||
|
||||
// Compute the hash modulo for the dependency derivation
|
||||
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
|
||||
ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto depDrvHash = depHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the dependency in the substituter
|
||||
auto depOutputPath = substituter->addToStore(
|
||||
"dep-drv-out",
|
||||
@@ -311,7 +322,7 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
|
||||
// Add the realisation for the dependency to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
depDrvPath,
|
||||
depDrvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
@@ -342,6 +353,11 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("issue-11928/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo for the root derivation
|
||||
auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
|
||||
ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto rootDrvHash = rootHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the root derivation
|
||||
// Note: it does NOT reference the dependency's output
|
||||
auto rootOutputPath = substituter->addToStore(
|
||||
@@ -362,12 +378,12 @@ TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// The DrvOutputs for both derivations
|
||||
DrvOutput depDrvOutput{depDrvPath, "out"};
|
||||
DrvOutput rootDrvOutput{rootDrvPath, "out"};
|
||||
DrvOutput depDrvOutput{depDrvHash, "out"};
|
||||
DrvOutput rootDrvOutput{rootDrvHash, "out"};
|
||||
|
||||
// Add the realisation for the root derivation to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
rootDrvPath,
|
||||
rootDrvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
|
||||
@@ -514,7 +514,7 @@ StorePath BinaryCacheStore::addToStore(
|
||||
|
||||
std::string BinaryCacheStore::makeRealisationPath(const DrvOutput & id)
|
||||
{
|
||||
return realisationsPrefix + "/" + id.drvPath.to_string() + "/" + id.outputName + ".doi";
|
||||
return realisationsPrefix + "/" + id.to_string() + ".doi";
|
||||
}
|
||||
|
||||
void BinaryCacheStore::queryRealisationUncached(
|
||||
@@ -535,10 +535,7 @@ void BinaryCacheStore::queryRealisationUncached(
|
||||
realisation = std::make_shared<const UnkeyedRealisation>(nlohmann::json::parse(*data));
|
||||
} catch (Error & e) {
|
||||
e.addTrace(
|
||||
{},
|
||||
"while parsing file '%s' as a build trace value for key '%s'",
|
||||
outputInfoFilePath,
|
||||
id.to_string());
|
||||
{}, "while parsing file '%s' as a realisation for key '%s'", outputInfoFilePath, id.to_string());
|
||||
throw;
|
||||
}
|
||||
return (*callbackPtr)(std::move(realisation));
|
||||
@@ -554,10 +551,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
|
||||
{
|
||||
if (diskCache)
|
||||
diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info);
|
||||
upsertFile(
|
||||
makeRealisationPath(info.id),
|
||||
static_cast<nlohmann::json>(static_cast<const UnkeyedRealisation &>(info)).dump(),
|
||||
"application/json");
|
||||
upsertFile(makeRealisationPath(info.id), static_cast<nlohmann::json>(info).dump(), "application/json");
|
||||
}
|
||||
|
||||
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
|
||||
|
||||
@@ -301,8 +301,9 @@ Goal::Co DerivationBuildingGoal::tryToBuild(StorePathSet inputPaths)
|
||||
given this information by the downstream goal, that cannot happen
|
||||
anymore if the downstream goal only cares about one output, but
|
||||
we care about all outputs. */
|
||||
for (auto & [outputName, _] : drv->outputs) {
|
||||
InitialOutput v;
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
for (auto & [outputName, outputHash] : outputHashes) {
|
||||
InitialOutput v{.outputHash = outputHash};
|
||||
|
||||
/* TODO we might want to also allow randomizing the paths
|
||||
for regular CA derivations, e.g. for sake of checking
|
||||
@@ -1267,7 +1268,7 @@ DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> &
|
||||
: PathStatus::Corrupt,
|
||||
};
|
||||
}
|
||||
auto drvOutput = DrvOutput{drvPath, i.first};
|
||||
auto drvOutput = DrvOutput{info.outputHash, i.first};
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
if (auto real = worker.store.queryRealisation(drvOutput)) {
|
||||
info.known = {
|
||||
|
||||
@@ -37,6 +37,12 @@ DerivationGoal::DerivationGoal(
|
||||
, drvPath(drvPath)
|
||||
, wantedOutput(wantedOutput)
|
||||
, drv{std::make_unique<Derivation>(drv)}
|
||||
, outputHash{[&] {
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, drv);
|
||||
if (auto * mOutputHash = get(outputHashes, wantedOutput))
|
||||
return *mOutputHash;
|
||||
throw Error("derivation '%s' does not have output '%s'", worker.store.printStorePath(drvPath), wantedOutput);
|
||||
}()}
|
||||
, buildMode(buildMode)
|
||||
{
|
||||
|
||||
@@ -96,7 +102,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
them. */
|
||||
if (worker.settings.useSubstitutes && drvOptions.substitutesAllowed(worker.settings)) {
|
||||
if (!checkResult) {
|
||||
DrvOutput id{drvPath, wantedOutput};
|
||||
DrvOutput id{outputHash, wantedOutput};
|
||||
auto g = worker.makeDrvOutputSubstitutionGoal(id);
|
||||
waitees.insert(g);
|
||||
co_await await(std::move(waitees));
|
||||
@@ -180,7 +186,12 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
// No `std::visit` for coroutines yet
|
||||
if (auto * successP = resolvedResult.tryGetSuccess()) {
|
||||
auto & success = *successP;
|
||||
if (!drv->outputs.contains(wantedOutput))
|
||||
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
|
||||
auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
|
||||
|
||||
auto outputHash = get(outputHashes, wantedOutput);
|
||||
auto resolvedHash = get(resolvedHashes, wantedOutput);
|
||||
if ((!outputHash) || (!resolvedHash))
|
||||
throw Error(
|
||||
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)",
|
||||
worker.store.printStorePath(drvPath),
|
||||
@@ -189,7 +200,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
auto realisation = [&] {
|
||||
auto take1 = get(success.builtOutputs, wantedOutput);
|
||||
if (take1)
|
||||
return *take1;
|
||||
return static_cast<UnkeyedRealisation>(*take1);
|
||||
|
||||
/* The above `get` should work. But stateful tracking of
|
||||
outputs in resolvedResult, this can get out of sync with the
|
||||
@@ -197,7 +208,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
check the store directly if it fails. */
|
||||
auto take2 = worker.evalStore.queryRealisation(
|
||||
DrvOutput{
|
||||
.drvPath = pathResolved,
|
||||
.drvHash = *resolvedHash,
|
||||
.outputName = wantedOutput,
|
||||
});
|
||||
if (take2)
|
||||
@@ -213,7 +224,7 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
Realisation newRealisation{
|
||||
realisation,
|
||||
{
|
||||
.drvPath = drvPath,
|
||||
.drvHash = *outputHash,
|
||||
.outputName = wantedOutput,
|
||||
}};
|
||||
newRealisation.signatures.clear();
|
||||
@@ -259,7 +270,16 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
/* In checking mode, the builder will not register any outputs.
|
||||
So we want to make sure the ones that we wanted to check are
|
||||
properly there. */
|
||||
success.builtOutputs = {{wantedOutput, assertPathValidity()}};
|
||||
success.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
assertPathValidity(),
|
||||
{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}};
|
||||
} else {
|
||||
/* Otherwise the builder will give us info for out output, but
|
||||
also for other outputs. Filter down to just our output so as
|
||||
@@ -278,7 +298,16 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
if (success.builtOutputs.count(wantedOutput) == 0) {
|
||||
debug(
|
||||
"BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput);
|
||||
success.builtOutputs = {{wantedOutput, assertPathValidity()}};
|
||||
success.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
assertPathValidity(),
|
||||
{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -375,7 +404,7 @@ std::optional<std::pair<UnkeyedRealisation, PathStatus>> DerivationGoal::checkPa
|
||||
if (drv->type().isImpure())
|
||||
return std::nullopt;
|
||||
|
||||
auto drvOutput = DrvOutput{drvPath, wantedOutput};
|
||||
auto drvOutput = DrvOutput{outputHash, wantedOutput};
|
||||
|
||||
std::optional<UnkeyedRealisation> mRealisation;
|
||||
|
||||
@@ -415,7 +444,7 @@ std::optional<std::pair<UnkeyedRealisation, PathStatus>> DerivationGoal::checkPa
|
||||
Realisation{
|
||||
*mRealisation,
|
||||
{
|
||||
.drvPath = drvPath,
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
});
|
||||
@@ -446,7 +475,16 @@ Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Unke
|
||||
return Goal::doneSuccess(
|
||||
BuildResult::Success{
|
||||
.status = status,
|
||||
.builtOutputs = {{wantedOutput, std::move(builtOutput)}},
|
||||
.builtOutputs = {{
|
||||
wantedOutput,
|
||||
{
|
||||
std::move(builtOutput),
|
||||
DrvOutput{
|
||||
.drvHash = outputHash,
|
||||
.outputName = wantedOutput,
|
||||
},
|
||||
},
|
||||
}},
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput & id, Worke
|
||||
: Goal(worker, init())
|
||||
, id(id)
|
||||
{
|
||||
name = fmt("substitution of '%s'", id.render(worker.store));
|
||||
name = fmt("substitution of '%s'", id.to_string());
|
||||
trace("created");
|
||||
}
|
||||
|
||||
@@ -93,8 +93,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
|
||||
/* None left. Terminate this goal and let someone else deal
|
||||
with it. */
|
||||
debug(
|
||||
"derivation output '%s' is required, but there is no substituter that can provide it", id.render(worker.store));
|
||||
debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string());
|
||||
|
||||
if (substituterFailed) {
|
||||
worker.failedSubstitutions++;
|
||||
@@ -109,7 +108,7 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
|
||||
std::string DrvOutputSubstitutionGoal::key()
|
||||
{
|
||||
return "a$" + std::string(id.render(worker.store));
|
||||
return "a$" + std::string(id.to_string());
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -2,16 +2,7 @@
|
||||
-- Won't be loaded unless the experimental feature `ca-derivations`
|
||||
-- is enabled
|
||||
|
||||
-- Why the `*V<N>` tables
|
||||
--
|
||||
-- We are trying to keep different versions of the experiment to have
|
||||
-- completely independent extra schemas from one another. This will
|
||||
-- enable people to switch between versions of the experiment (including
|
||||
-- newer to older) without migrating between them, but at the cost
|
||||
-- of having many abandoned tables lying around. Closer to the end of
|
||||
-- the experiment, we'll provide guidance on how to clean this up.
|
||||
|
||||
create table if not exists BuildTraceV2 (
|
||||
create table if not exists Realisations (
|
||||
id integer primary key autoincrement not null,
|
||||
drvPath text not null,
|
||||
outputName text not null, -- symbolic output id, usually "out"
|
||||
@@ -20,4 +11,31 @@ create table if not exists BuildTraceV2 (
|
||||
foreign key (outputPath) references ValidPaths(id) on delete cascade
|
||||
);
|
||||
|
||||
create index if not exists IndexBuildTraceV2 on BuildTraceV2(drvPath, outputName);
|
||||
create index if not exists IndexRealisations on Realisations(drvPath, outputName);
|
||||
|
||||
-- We can end-up in a weird edge-case where a path depends on itself because
|
||||
-- it’s an output of a CA derivation, that happens to be the same as one of its
|
||||
-- dependencies.
|
||||
-- In that case we have a dependency loop (path -> realisation1 -> realisation2
|
||||
-- -> path) that we need to break by removing the dependencies between the
|
||||
-- realisations
|
||||
create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths
|
||||
begin
|
||||
delete from RealisationsRefs where realisationReference in (
|
||||
select id from Realisations where outputPath = old.id
|
||||
);
|
||||
end;
|
||||
|
||||
create table if not exists RealisationsRefs (
|
||||
referrer integer not null,
|
||||
realisationReference integer,
|
||||
foreign key (referrer) references Realisations(id) on delete cascade,
|
||||
foreign key (realisationReference) references Realisations(id) on delete restrict
|
||||
);
|
||||
-- used by deletion trigger
|
||||
create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference);
|
||||
|
||||
-- used by QueryRealisationReferences
|
||||
create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
|
||||
-- used by cascade deletion when ValidPaths is deleted
|
||||
create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);
|
||||
|
||||
@@ -47,6 +47,34 @@ void CommonProto::Serialise<ContentAddress>::write(
|
||||
conn.to << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
Realisation CommonProto::Serialise<Realisation>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
std::string rawInput = readString(conn.from);
|
||||
try {
|
||||
return nlohmann::json::parse(rawInput);
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "while parsing a realisation object in the remote protocol");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<Realisation>::write(
|
||||
const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation)
|
||||
{
|
||||
conn.to << static_cast<nlohmann::json>(realisation).dump();
|
||||
}
|
||||
|
||||
DrvOutput CommonProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return DrvOutput::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<DrvOutput>::write(
|
||||
const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
|
||||
{
|
||||
conn.to << drvOutput.to_string();
|
||||
}
|
||||
|
||||
std::optional<StorePath>
|
||||
CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
|
||||
@@ -955,8 +955,14 @@ static void performOp(
|
||||
|
||||
case WorkerProto::Op::RegisterDrvOutput: {
|
||||
logger->startWork();
|
||||
auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
|
||||
store->registerDrvOutput(realisation);
|
||||
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
|
||||
auto outputPath = StorePath(readString(conn.from));
|
||||
store->registerDrvOutput(Realisation{{.outPath = outputPath}, outputId});
|
||||
} else {
|
||||
auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn);
|
||||
store->registerDrvOutput(realisation);
|
||||
}
|
||||
logger->stopWork();
|
||||
break;
|
||||
}
|
||||
@@ -964,13 +970,19 @@ static void performOp(
|
||||
case WorkerProto::Op::QueryRealisation: {
|
||||
logger->startWork();
|
||||
auto outputId = WorkerProto::Serialise<DrvOutput>::read(*store, rconn);
|
||||
std::optional<UnkeyedRealisation> info = *store->queryRealisation(outputId);
|
||||
auto info = store->queryRealisation(outputId);
|
||||
logger->stopWork();
|
||||
/* Only return the new format because if we got past
|
||||
`DrvOutput` serialization, we know that is what we're using.
|
||||
*/
|
||||
assert(conn.protoVersion.features.contains(WorkerProto::featureRealisationWithPath));
|
||||
WorkerProto::write(*store, wconn, info);
|
||||
if (conn.protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
std::set<StorePath> outPaths;
|
||||
if (info)
|
||||
outPaths.insert(info->outPath);
|
||||
WorkerProto::write(*store, wconn, outPaths);
|
||||
} else {
|
||||
std::set<Realisation> realisations;
|
||||
if (info)
|
||||
realisations.insert({*info, outputId});
|
||||
WorkerProto::write(*store, wconn, realisations);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -861,14 +861,13 @@ DrvHashes drvHashes;
|
||||
/* Look up the derivation by value and memoize the
|
||||
`hashDerivationModulo` call.
|
||||
*/
|
||||
static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||
{
|
||||
std::optional<DrvHashModulo> hash;
|
||||
std::optional<DrvHash> hash;
|
||||
if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) {
|
||||
return *hash;
|
||||
}
|
||||
auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false);
|
||||
|
||||
// Cache it
|
||||
drvHashes.insert_or_assign(drvPath, h);
|
||||
return h;
|
||||
@@ -891,10 +890,12 @@ static DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPa
|
||||
don't leak the provenance of fixed outputs, reducing pointless cache
|
||||
misses as the build itself won't know this.
|
||||
*/
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||
{
|
||||
auto type = drv.type();
|
||||
|
||||
/* Return a fixed hash for fixed-output derivations. */
|
||||
if (drv.type().isFixed()) {
|
||||
if (type.isFixed()) {
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & i : drv.outputs) {
|
||||
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
|
||||
@@ -904,66 +905,54 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
||||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||
}
|
||||
return outputHashes;
|
||||
return DrvHash{
|
||||
.hashes = outputHashes,
|
||||
.kind = DrvHash::Kind::Regular,
|
||||
};
|
||||
}
|
||||
|
||||
if (std::visit(
|
||||
overloaded{
|
||||
[](const DerivationType::InputAddressed & ia) {
|
||||
/* This might be a "pesimistically" deferred output, so we don't
|
||||
"taint" the kind yet. */
|
||||
return false;
|
||||
},
|
||||
[](const DerivationType::ContentAddressed & ca) {
|
||||
// Already covered
|
||||
assert(!ca.fixed);
|
||||
return true;
|
||||
},
|
||||
[](const DerivationType::Impure &) { return true; }},
|
||||
drv.type().raw)) {
|
||||
return DrvHashModulo::DeferredDrv{};
|
||||
}
|
||||
auto kind = std::visit(
|
||||
overloaded{
|
||||
[](const DerivationType::InputAddressed & ia) {
|
||||
/* This might be a "pesimistically" deferred output, so we don't
|
||||
"taint" the kind yet. */
|
||||
return DrvHash::Kind::Regular;
|
||||
},
|
||||
[](const DerivationType::ContentAddressed & ca) {
|
||||
return ca.fixed ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred;
|
||||
},
|
||||
[](const DerivationType::Impure &) -> DrvHash::Kind { return DrvHash::Kind::Deferred; }},
|
||||
drv.type().raw);
|
||||
|
||||
/* For other derivations, replace the inputs paths with recursive
|
||||
calls to this function. */
|
||||
DerivedPathMap<StringSet>::ChildNode::Map inputs2;
|
||||
for (auto & [drvPath, node] : drv.inputDrvs.map) {
|
||||
/* Need to build and resolve dynamic derivations first */
|
||||
if (!node.childMap.empty()) {
|
||||
return DrvHashModulo::DeferredDrv{};
|
||||
}
|
||||
|
||||
const auto & res = pathDerivationModulo(store, drvPath);
|
||||
if (std::visit(
|
||||
overloaded{
|
||||
[&](const DrvHashModulo::DeferredDrv &) { return true; },
|
||||
// Regular non-CA derivation, replace derivation
|
||||
[&](const DrvHashModulo::DrvHash & drvHash) {
|
||||
inputs2.insert_or_assign(drvHash.to_string(HashFormat::Base16, false), node);
|
||||
return false;
|
||||
},
|
||||
// CA derivation's output hashes
|
||||
[&](const DrvHashModulo::CaOutputHashes & outputHashes) {
|
||||
for (auto & outputName : node.value) {
|
||||
/* Put each one in with a single "out" output.. */
|
||||
const auto h = get(outputHashes, outputName);
|
||||
if (!h)
|
||||
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
|
||||
inputs2.insert_or_assign(
|
||||
h->to_string(HashFormat::Base16, false),
|
||||
DerivedPathMap<StringSet>::ChildNode{
|
||||
.value = {"out"},
|
||||
});
|
||||
}
|
||||
return false;
|
||||
},
|
||||
},
|
||||
res.raw)) {
|
||||
return DrvHashModulo::DeferredDrv{};
|
||||
if (res.kind == DrvHash::Kind::Deferred)
|
||||
kind = DrvHash::Kind::Deferred;
|
||||
for (auto & outputName : node.value) {
|
||||
const auto h = get(res.hashes, outputName);
|
||||
if (!h)
|
||||
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
|
||||
inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName);
|
||||
}
|
||||
}
|
||||
|
||||
return hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||
|
||||
std::map<std::string, Hash> outputHashes;
|
||||
for (const auto & [outputName, _] : drv.outputs) {
|
||||
outputHashes.insert_or_assign(outputName, hash);
|
||||
}
|
||||
|
||||
return DrvHash{
|
||||
.hashes = outputHashes,
|
||||
.kind = kind,
|
||||
};
|
||||
}
|
||||
|
||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
|
||||
{
|
||||
return hashDerivationModulo(store, drv, true).hashes;
|
||||
}
|
||||
|
||||
static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store)
|
||||
@@ -1113,6 +1102,26 @@ void BasicDerivation::applyRewrites(const StringMap & rewrites)
|
||||
}
|
||||
}
|
||||
|
||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||
{
|
||||
drv.applyRewrites(rewrites);
|
||||
|
||||
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
|
||||
for (auto & [outputName, output] : drv.outputs) {
|
||||
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) {
|
||||
auto h = get(hashModulo.hashes, outputName);
|
||||
if (!h)
|
||||
throw Error(
|
||||
"derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", drv.name, outputName);
|
||||
auto outPath = store.makeOutputPath(outputName, *h, drv.name);
|
||||
drv.env[outputName] = store.printStorePath(outPath);
|
||||
output = DerivationOutput::InputAddressed{
|
||||
.path = std::move(outPath),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Derivation::shouldResolve() const
|
||||
{
|
||||
/* No input drvs means nothing to resolve. */
|
||||
@@ -1224,13 +1233,9 @@ std::optional<BasicDerivation> Derivation::tryResolve(
|
||||
queryResolutionChain))
|
||||
return std::nullopt;
|
||||
|
||||
resolved.applyRewrites(inputRewrites);
|
||||
rewriteDerivation(store, resolved, inputRewrites);
|
||||
|
||||
Derivation resolved2{std::move(resolved)};
|
||||
|
||||
resolved2.fillInOutputPaths(store);
|
||||
|
||||
return resolved2;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1257,15 +1262,7 @@ std::optional<BasicDerivation> Derivation::tryResolve(
|
||||
template<bool fillIn>
|
||||
static void processDerivationOutputPaths(Store & store, auto && drv, std::string_view drvName)
|
||||
{
|
||||
std::optional<DrvHashModulo> hashModulo_;
|
||||
|
||||
auto hashModulo = [&]() -> const auto & {
|
||||
if (!hashModulo_) {
|
||||
// somewhat expensive so we do lazily
|
||||
hashModulo_ = hashDerivationModulo(store, drv, true);
|
||||
}
|
||||
return *hashModulo_;
|
||||
};
|
||||
std::optional<DrvHash> hashesModulo;
|
||||
|
||||
for (auto & [outputName, output] : drv.outputs) {
|
||||
auto envHasRightPath = [&](const StorePath & actual, bool isDeferred = false) {
|
||||
@@ -1302,64 +1299,65 @@ static void processDerivationOutputPaths(Store & store, auto && drv, std::string
|
||||
}
|
||||
};
|
||||
auto hash = [&]<typename Output>(const Output & outputVariant) {
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const DrvHashModulo::DrvHash & drvHash) {
|
||||
auto outPath = store.makeOutputPath(outputName, drvHash, drvName);
|
||||
if (!hashesModulo) {
|
||||
// somewhat expensive so we do lazily
|
||||
hashesModulo = hashDerivationModulo(store, drv, true);
|
||||
}
|
||||
switch (hashesModulo->kind) {
|
||||
case DrvHash::Kind::Regular: {
|
||||
auto h = get(hashesModulo->hashes, outputName);
|
||||
if (!h)
|
||||
throw Error("derivation produced no hash for output '%s'", outputName);
|
||||
auto outPath = store.makeOutputPath(outputName, *h, drvName);
|
||||
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
if (outputVariant.path == outPath) {
|
||||
envHasRightPath(outPath);
|
||||
return; // Correct case
|
||||
}
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be '%s'",
|
||||
store.printStorePath(outputVariant.path),
|
||||
store.printStorePath(outPath));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
if constexpr (fillIn) {
|
||||
/* Fill in output path for Deferred outputs */
|
||||
output = DerivationOutput::InputAddressed{
|
||||
.path = outPath,
|
||||
};
|
||||
envHasRightPath(outPath);
|
||||
} else {
|
||||
/* Validation mode: deferred outputs
|
||||
should have been filled in */
|
||||
warn(
|
||||
"derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.",
|
||||
store.printStorePath(outPath));
|
||||
}
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
},
|
||||
[&](const DrvHashModulo::CaOutputHashes &) {
|
||||
/* Shouldn't happen as the original output is
|
||||
input-addressed (or deferred waiting to be). */
|
||||
assert(false);
|
||||
},
|
||||
[&](const DrvHashModulo::DeferredDrv &) {
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be deferred",
|
||||
store.printStorePath(outputVariant.path));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
/* Correct: Deferred output with Deferred hash kind. */
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
},
|
||||
},
|
||||
hashModulo().raw);
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
if (outputVariant.path == outPath) {
|
||||
return; // Correct case
|
||||
}
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be '%s'",
|
||||
store.printStorePath(outputVariant.path),
|
||||
store.printStorePath(outPath));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
if constexpr (fillIn)
|
||||
/* Fill in output path for Deferred
|
||||
outputs */
|
||||
output = DerivationOutput::InputAddressed{
|
||||
.path = outPath,
|
||||
};
|
||||
else
|
||||
/* Validation mode: deferred outputs
|
||||
should have been filled in */
|
||||
warn(
|
||||
"derivation has incorrect deferred output, should be '%s'.\nThis will be an error in future versions of Nix; compatibility of CA derivations will be broken.",
|
||||
store.printStorePath(outPath));
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
envHasRightPath(outPath);
|
||||
break;
|
||||
}
|
||||
case DrvHash::Kind::Deferred:
|
||||
if constexpr (std::is_same_v<Output, DerivationOutput::InputAddressed>) {
|
||||
/* Error case, an explicitly wrong path is
|
||||
always an error. */
|
||||
throw Error(
|
||||
"derivation has incorrect output '%s', should be deferred",
|
||||
store.printStorePath(outputVariant.path));
|
||||
} else if constexpr (std::is_same_v<Output, DerivationOutput::Deferred>) {
|
||||
/* Correct: Deferred output with Deferred
|
||||
hash kind. */
|
||||
} else {
|
||||
/* Will never happen, based on where
|
||||
`hash` is called. */
|
||||
static_assert(false);
|
||||
}
|
||||
break;
|
||||
}
|
||||
};
|
||||
std::visit(
|
||||
overloaded{
|
||||
|
||||
@@ -330,7 +330,7 @@ struct DummyStoreImpl : DummyStore
|
||||
|
||||
void registerDrvOutput(const Realisation & output) override
|
||||
{
|
||||
buildTrace.insert_or_visit({output.id.drvPath, {{output.id.outputName, output}}}, [&](auto & kv) {
|
||||
buildTrace.insert_or_visit({output.id.drvHash, {{output.id.outputName, output}}}, [&](auto & kv) {
|
||||
kv.second.insert_or_assign(output.id.outputName, output);
|
||||
});
|
||||
}
|
||||
@@ -339,7 +339,7 @@ struct DummyStoreImpl : DummyStore
|
||||
const DrvOutput & drvOutput, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
|
||||
{
|
||||
bool visited = false;
|
||||
buildTrace.cvisit(drvOutput.drvPath, [&](const auto & kv) {
|
||||
buildTrace.cvisit(drvOutput.drvHash, [&](const auto & kv) {
|
||||
if (auto it = kv.second.find(drvOutput.outputName); it != kv.second.end()) {
|
||||
visited = true;
|
||||
callback(std::make_shared<UnkeyedRealisation>(it->second));
|
||||
@@ -436,7 +436,11 @@ ref<DummyStore> adl_serializer<ref<DummyStore>>::from_json(const json & json)
|
||||
for (auto & [k1, v2] : getObject(v)) {
|
||||
UnkeyedRealisation realisation = v2;
|
||||
res->buildTrace.insert_or_visit(
|
||||
{StorePath{k0}, {{k1, realisation}}}, [&](auto & kv) { kv.second.insert_or_assign(k1, realisation); });
|
||||
{
|
||||
Hash::parseExplicitFormatUnprefixed(k0, HashAlgorithm::SHA256, HashFormat::Base64),
|
||||
{{k1, realisation}},
|
||||
},
|
||||
[&](auto & kv) { kv.second.insert_or_assign(k1, realisation); });
|
||||
}
|
||||
}
|
||||
return res;
|
||||
@@ -469,7 +473,7 @@ void adl_serializer<DummyStore>::to_json(json & json, const DummyStore & val)
|
||||
auto obj = json::object();
|
||||
val.buildTrace.cvisit_all([&](const auto & kv) {
|
||||
auto & [k, v] = kv;
|
||||
auto & obj2 = obj[k.to_string()] = json::object();
|
||||
auto & obj2 = obj[k.to_string(HashFormat::Base64, false)] = json::object();
|
||||
for (auto & [k2, v2] : kv.second)
|
||||
obj2[k2] = v2;
|
||||
});
|
||||
|
||||
@@ -90,18 +90,8 @@ protected:
|
||||
|
||||
/**
|
||||
* The prefix under which realisation infos will be stored
|
||||
*
|
||||
* @note The previous (still experimental, though) hash-keyed
|
||||
* realisations were under "realisations". "build trace" is a better
|
||||
* name anyways (issue #11895). This is call "v2" accordingly.
|
||||
*
|
||||
* While we're experimenting, we'll freely increase this version
|
||||
* number. Old build traces will just be "abandoned" at the old URL.
|
||||
* When we are done experimenting, we'll try lean more on versioning
|
||||
* the build trace entries themselves than the entire directory, for
|
||||
* a smoother migration path.
|
||||
*/
|
||||
constexpr const static std::string realisationsPrefix = "build-trace-v2";
|
||||
constexpr const static std::string realisationsPrefix = "realisations";
|
||||
|
||||
constexpr const static std::string cacheInfoFile = "nix-cache-info";
|
||||
|
||||
@@ -110,7 +100,7 @@ protected:
|
||||
/**
|
||||
* Compute the path to the given realisation
|
||||
*
|
||||
* It's `${realisationsPrefix}/${drvPath}/${outputName}`.
|
||||
* It's `${realisationsPrefix}/${drvOutput}.doi`.
|
||||
*/
|
||||
std::string makeRealisationPath(const DrvOutput & id);
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ struct InitialOutputStatus
|
||||
|
||||
struct InitialOutput
|
||||
{
|
||||
Hash outputHash;
|
||||
std::optional<InitialOutputStatus> known;
|
||||
};
|
||||
|
||||
|
||||
@@ -66,6 +66,8 @@ private:
|
||||
*/
|
||||
std::unique_ptr<Derivation> drv;
|
||||
|
||||
const Hash outputHash;
|
||||
|
||||
const BuildMode buildMode;
|
||||
|
||||
/**
|
||||
|
||||
@@ -26,13 +26,12 @@ namespace nix {
|
||||
LengthPrefixedProtoHelper<CommonProto, T>::write(store, conn, t); \
|
||||
}
|
||||
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
|
||||
#define COMMA_ ,
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T COMMA_ typename Compare>, std::set<T COMMA_ Compare>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K COMMA_ typename V COMMA_ typename Compare>, std::map<K COMMA_ V COMMA_ Compare>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename K COMMA_ typename V>, std::map<K COMMA_ V>)
|
||||
#undef COMMA_
|
||||
|
||||
/* protocol-specific templates */
|
||||
|
||||
@@ -88,9 +88,8 @@ DECLARE_COMMON_SERIALISER(std::set<T COMMA_ Compare>);
|
||||
template<typename... Ts>
|
||||
DECLARE_COMMON_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
template<typename K, typename V, typename Compare>
|
||||
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
|
||||
#undef COMMA_
|
||||
template<typename K, typename V>
|
||||
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V>);
|
||||
|
||||
/**
|
||||
* These use the empty string for the null case, relying on the fact
|
||||
|
||||
@@ -504,40 +504,35 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName);
|
||||
* derivations (fixed-output or not) will have a different hash for each
|
||||
* output.
|
||||
*/
|
||||
struct DrvHashModulo
|
||||
struct DrvHash
|
||||
{
|
||||
/**
|
||||
* Single hash for the derivation
|
||||
*
|
||||
* This is for an input-addressed derivation that doesn't
|
||||
* transitively depend on any floating-CA derivations.
|
||||
* Map from output names to hashes
|
||||
*/
|
||||
using DrvHash = Hash;
|
||||
std::map<std::string, Hash> hashes;
|
||||
|
||||
enum struct Kind : bool {
|
||||
/**
|
||||
* Statically determined derivations.
|
||||
* This hash will be directly used to compute the output paths
|
||||
*/
|
||||
Regular,
|
||||
|
||||
/**
|
||||
* Floating-output derivations (and their reverse dependencies).
|
||||
*/
|
||||
Deferred,
|
||||
};
|
||||
|
||||
/**
|
||||
* Known CA drv's output hashes, for fixed-output derivations whose
|
||||
* output hashes are always known since they are fixed up-front.
|
||||
* The kind of derivation this is, simplified for just "derivation hash
|
||||
* modulo" purposes.
|
||||
*/
|
||||
using CaOutputHashes = std::map<std::string, Hash>;
|
||||
|
||||
/**
|
||||
* This derivation doesn't yet have known output hashes.
|
||||
*
|
||||
* Either because itself is floating CA, or it (transtively) depends
|
||||
* on a floating CA derivation.
|
||||
*/
|
||||
using DeferredDrv = std::monostate;
|
||||
|
||||
using Raw = std::variant<DrvHash, CaOutputHashes, DeferredDrv>;
|
||||
|
||||
Raw raw;
|
||||
|
||||
bool operator==(const DrvHashModulo &) const = default;
|
||||
// auto operator <=> (const DrvHashModulo &) const = default;
|
||||
|
||||
MAKE_WRAPPER_CONSTRUCTOR(DrvHashModulo);
|
||||
Kind kind;
|
||||
};
|
||||
|
||||
void operator|=(DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
|
||||
|
||||
/**
|
||||
* Returns hashes with the details of fixed-output subderivations
|
||||
* expunged.
|
||||
@@ -562,17 +557,15 @@ struct DrvHashModulo
|
||||
* ATerm, after subderivations have been likewise expunged from that
|
||||
* derivation.
|
||||
*/
|
||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||
|
||||
/**
|
||||
* If a derivation is input addressed and doesn't yet have its input
|
||||
* addressed (is deferred) try using `hashDerivationModulo`.
|
||||
* Return a map associating each output to a hash that uniquely identifies its
|
||||
* derivation (modulo the self-references).
|
||||
*
|
||||
* Does nothing if not deferred input-addressed, or
|
||||
* `hashDerivationModulo` indicates it is missing inputs' output paths
|
||||
* and is not yet ready (and must stay deferred).
|
||||
* \todo What is the Hash in this map?
|
||||
*/
|
||||
void resolveInputAddressed(Store & store, Derivation & drv);
|
||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
|
||||
|
||||
struct DrvHashFct
|
||||
{
|
||||
@@ -587,7 +580,7 @@ struct DrvHashFct
|
||||
/**
|
||||
* Memoisation of hashDerivationModulo().
|
||||
*/
|
||||
typedef boost::concurrent_flat_map<StorePath, DrvHashModulo, DrvHashFct> DrvHashes;
|
||||
typedef boost::concurrent_flat_map<StorePath, DrvHash, DrvHashFct> DrvHashes;
|
||||
|
||||
// FIXME: global, though at least thread-safe.
|
||||
extern DrvHashes drvHashes;
|
||||
|
||||
@@ -49,7 +49,7 @@ struct DummyStore : virtual Store
|
||||
* outer map for the derivation, and inner maps for the outputs of a
|
||||
* given derivation.
|
||||
*/
|
||||
boost::concurrent_flat_map<StorePath, std::map<std::string, UnkeyedRealisation>> buildTrace;
|
||||
boost::concurrent_flat_map<Hash, std::map<std::string, UnkeyedRealisation>> buildTrace;
|
||||
|
||||
DummyStore(ref<const Config> config)
|
||||
: Store{*config}
|
||||
|
||||
@@ -56,14 +56,14 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector<T>);
|
||||
#define COMMA_ ,
|
||||
template<class Inner, typename T, typename Compare>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T COMMA_ Compare>);
|
||||
#undef COMMA_
|
||||
|
||||
template<class Inner, typename... Ts>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>);
|
||||
|
||||
template<class Inner, typename K, typename V, typename Compare>
|
||||
#define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V, Compare>
|
||||
template<class Inner, typename K, typename V>
|
||||
#define LENGTH_PREFIXED_PROTO_HELPER_X std::map<K, V>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, LENGTH_PREFIXED_PROTO_HELPER_X);
|
||||
#undef COMMA_
|
||||
|
||||
template<class Inner, typename T>
|
||||
std::vector<T>
|
||||
@@ -109,11 +109,11 @@ void LengthPrefixedProtoHelper<Inner, std::set<T, Compare>>::write(
|
||||
}
|
||||
}
|
||||
|
||||
template<class Inner, typename K, typename V, typename Compare>
|
||||
std::map<K, V, Compare> LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::read(
|
||||
const StoreDirConfig & store, typename Inner::ReadConn conn)
|
||||
template<class Inner, typename K, typename V>
|
||||
std::map<K, V>
|
||||
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(const StoreDirConfig & store, typename Inner::ReadConn conn)
|
||||
{
|
||||
std::map<K, V, Compare> resMap;
|
||||
std::map<K, V> resMap;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
auto k = S<K>::read(store, conn);
|
||||
@@ -123,9 +123,9 @@ std::map<K, V, Compare> LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>
|
||||
return resMap;
|
||||
}
|
||||
|
||||
template<class Inner, typename K, typename V, typename Compare>
|
||||
void LengthPrefixedProtoHelper<Inner, std::map<K, V, Compare>>::write(
|
||||
const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V, Compare> & resMap)
|
||||
template<class Inner, typename K, typename V>
|
||||
void LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
|
||||
const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
|
||||
{
|
||||
conn.to << resMap.size();
|
||||
for (auto & i : resMap) {
|
||||
|
||||
@@ -18,40 +18,33 @@ struct OutputsSpec;
|
||||
/**
|
||||
* A general `Realisation` key.
|
||||
*
|
||||
* This is similar to a `DerivedPath::Built`, except it is only a single
|
||||
* step: `drvPath` is a `StorePath` rather than a `DerivedPath`.
|
||||
* This is similar to a `DerivedPath::Opaque`, but the derivation is
|
||||
* identified by its "hash modulo" instead of by its store path.
|
||||
*/
|
||||
struct DrvOutput
|
||||
{
|
||||
/**
|
||||
* The store path to the derivation
|
||||
* The hash modulo of the derivation.
|
||||
*
|
||||
* Computed from the derivation itself for most types of
|
||||
* derivations, but computed from the (fixed) content address of the
|
||||
* output for fixed-output derivations.
|
||||
*/
|
||||
StorePath drvPath;
|
||||
Hash drvHash;
|
||||
|
||||
/**
|
||||
* The name of the output.
|
||||
*/
|
||||
OutputName outputName;
|
||||
|
||||
/**
|
||||
* Skips the store dir on the `drvPath`
|
||||
*/
|
||||
std::string to_string() const;
|
||||
|
||||
/**
|
||||
* Skips the store dir on the `drvPath`
|
||||
*/
|
||||
static DrvOutput from_string(std::string_view);
|
||||
std::string strHash() const
|
||||
{
|
||||
return drvHash.to_string(HashFormat::Base16, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Includes the store dir on `drvPath`
|
||||
*/
|
||||
std::string render(const StoreDirConfig & store) const;
|
||||
|
||||
/**
|
||||
* Includes the store dir on `drvPath`
|
||||
*/
|
||||
static DrvOutput parse(const StoreDirConfig & store, std::string_view);
|
||||
static DrvOutput parse(const std::string &);
|
||||
|
||||
bool operator==(const DrvOutput &) const = default;
|
||||
auto operator<=>(const DrvOutput &) const = default;
|
||||
@@ -71,16 +64,6 @@ struct UnkeyedRealisation
|
||||
|
||||
size_t checkSignatures(const DrvOutput & key, const PublicKeys & publicKeys) const;
|
||||
|
||||
/**
|
||||
* Just check the `outPath`. Signatures don't matter for this.
|
||||
* Callers must ensure that the corresponding key is the same for
|
||||
* most use-cases.
|
||||
*/
|
||||
bool isCompatibleWith(const UnkeyedRealisation & other) const
|
||||
{
|
||||
return outPath == other.outPath;
|
||||
}
|
||||
|
||||
const StorePath & getPath() const
|
||||
{
|
||||
return outPath;
|
||||
@@ -94,6 +77,8 @@ struct Realisation : UnkeyedRealisation
|
||||
{
|
||||
DrvOutput id;
|
||||
|
||||
bool isCompatibleWith(const UnkeyedRealisation & other) const;
|
||||
|
||||
bool operator==(const Realisation &) const = default;
|
||||
auto operator<=>(const Realisation &) const = default;
|
||||
};
|
||||
@@ -104,7 +89,16 @@ struct Realisation : UnkeyedRealisation
|
||||
* Since these are the outputs of a single derivation, we know the
|
||||
* output names are unique so we can use them as the map key.
|
||||
*/
|
||||
typedef std::map<OutputName, UnkeyedRealisation> SingleDrvOutputs;
|
||||
typedef std::map<OutputName, Realisation> SingleDrvOutputs;
|
||||
|
||||
/**
|
||||
* Collection type for multiple derivations' outputs' `Realisation`s.
|
||||
*
|
||||
* `DrvOutput` is used because in general the derivations are not all
|
||||
* the same, so we need to identify firstly which derivation, and
|
||||
* secondly which output of that derivation.
|
||||
*/
|
||||
typedef std::map<DrvOutput, Realisation> DrvOutputs;
|
||||
|
||||
struct OpaquePath
|
||||
{
|
||||
@@ -155,17 +149,19 @@ struct RealisedPath
|
||||
class MissingRealisation : public Error
|
||||
{
|
||||
public:
|
||||
MissingRealisation(const StoreDirConfig & store, DrvOutput & outputId)
|
||||
: MissingRealisation(store, outputId.drvPath, outputId.outputName)
|
||||
MissingRealisation(DrvOutput & outputId)
|
||||
: MissingRealisation(outputId.outputName, outputId.strHash())
|
||||
{
|
||||
}
|
||||
|
||||
MissingRealisation(const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName);
|
||||
MissingRealisation(
|
||||
const StoreDirConfig & store,
|
||||
const SingleDerivedPath & drvPath,
|
||||
const StorePath & drvPathResolved,
|
||||
const OutputName & outputName);
|
||||
MissingRealisation(std::string_view drv, OutputName outputName)
|
||||
: Error(
|
||||
"cannot operate on output '%s' of the "
|
||||
"unbuilt derivation '%s'",
|
||||
outputName,
|
||||
drv)
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -34,10 +34,8 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
#define SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
|
||||
SERVE_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA
|
||||
typename Compare>
|
||||
,
|
||||
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
|
||||
template<typename K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>,
|
||||
std::map<K SERVE_USE_LENGTH_PREFIX_SERIALISER_COMMA V>)
|
||||
|
||||
/**
|
||||
* Use `CommonProto` where possible.
|
||||
|
||||
@@ -8,18 +8,12 @@ namespace nix {
|
||||
#define SERVE_MAGIC_1 0x390c9deb
|
||||
#define SERVE_MAGIC_2 0x5452eecb
|
||||
|
||||
#define SERVE_PROTOCOL_VERSION (2 << 8 | 8)
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
struct StoreDirConfig;
|
||||
struct Source;
|
||||
|
||||
// items being serialised
|
||||
struct BuildResult;
|
||||
struct UnkeyedValidPathInfo;
|
||||
struct DrvOutput;
|
||||
struct UnkeyedRealisation;
|
||||
struct Realisation;
|
||||
|
||||
/**
|
||||
* The "serve protocol", used by ssh:// stores.
|
||||
@@ -69,7 +63,7 @@ struct ServeProto
|
||||
|
||||
static constexpr Version latest = {
|
||||
.major = 2,
|
||||
.minor = 8,
|
||||
.minor = 7,
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -211,12 +205,6 @@ inline std::ostream & operator<<(std::ostream & s, ServeProto::Command op)
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(BuildResult);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(DrvOutput);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(UnkeyedRealisation);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(Realisation);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo);
|
||||
template<>
|
||||
DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions);
|
||||
@@ -229,8 +217,8 @@ DECLARE_SERVE_SERIALISER(std::set<T COMMA_ Compare>);
|
||||
template<typename... Ts>
|
||||
DECLARE_SERVE_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
template<typename K, typename V, typename Compare>
|
||||
DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
|
||||
template<typename K, typename V>
|
||||
DECLARE_SERVE_SERIALISER(std::map<K COMMA_ V>);
|
||||
#undef COMMA_
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -34,10 +34,8 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
#define WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA ,
|
||||
WORKER_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA
|
||||
typename Compare>
|
||||
,
|
||||
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA Compare>)
|
||||
template<typename K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA typename V>,
|
||||
std::map<K WORKER_USE_LENGTH_PREFIX_SERIALISER_COMMA V>)
|
||||
|
||||
/**
|
||||
* Use `CommonProto` where possible.
|
||||
|
||||
@@ -12,12 +12,6 @@ namespace nix {
|
||||
#define WORKER_MAGIC_1 0x6e697863
|
||||
#define WORKER_MAGIC_2 0x6478696f
|
||||
|
||||
/* Note: you generally shouldn't change the protocol version. Define a
|
||||
new `WorkerProto::Feature` instead. */
|
||||
#define PROTOCOL_VERSION (1 << 8 | 39)
|
||||
#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18)
|
||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||
#define STDERR_NEXT 0x6f6c6d67
|
||||
#define STDERR_READ 0x64617461 // data needed from source
|
||||
#define STDERR_WRITE 0x64617416 // data for sink
|
||||
@@ -36,9 +30,6 @@ struct BuildResult;
|
||||
struct KeyedBuildResult;
|
||||
struct ValidPathInfo;
|
||||
struct UnkeyedValidPathInfo;
|
||||
struct DrvOutput;
|
||||
struct UnkeyedRealisation;
|
||||
struct Realisation;
|
||||
enum BuildMode : uint8_t;
|
||||
enum TrustedFlag : bool;
|
||||
enum class GCAction;
|
||||
@@ -118,12 +109,6 @@ struct WorkerProto
|
||||
|
||||
static const Version minimum;
|
||||
|
||||
/**
|
||||
* Feature for transmitting `UnkeyedRealisation` and `DrvOutput`
|
||||
* using drvPath (store path) instead of the old hash-based JSON format.
|
||||
*/
|
||||
static constexpr std::string_view featureRealisationWithPath = "realisation-with-path-not-hash";
|
||||
|
||||
/**
|
||||
* A unidirectional read connection, to be used by the read half of the
|
||||
* canonical serializers below.
|
||||
@@ -322,14 +307,6 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(DrvOutput);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(UnkeyedRealisation);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(Realisation);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(std::optional<UnkeyedRealisation>);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(BuildMode);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(GCAction);
|
||||
@@ -348,8 +325,8 @@ DECLARE_WORKER_SERIALISER(std::set<T COMMA_ Compare>);
|
||||
template<typename... Ts>
|
||||
DECLARE_WORKER_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
template<typename K, typename V, typename Compare>
|
||||
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V COMMA_ Compare>);
|
||||
template<typename K, typename V>
|
||||
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V>);
|
||||
#undef COMMA_
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -360,14 +360,14 @@ LocalStore::LocalStore(ref<const Config> config)
|
||||
state->stmts->RegisterRealisedOutput.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert into BuildTraceV2 (drvPath, outputName, outputPath, signatures)
|
||||
insert into Realisations (drvPath, outputName, outputPath, signatures)
|
||||
values (?, ?, (select id from ValidPaths where path = ?), ?)
|
||||
;
|
||||
)");
|
||||
state->stmts->UpdateRealisedOutput.create(
|
||||
state->db,
|
||||
R"(
|
||||
update BuildTraceV2
|
||||
update Realisations
|
||||
set signatures = ?
|
||||
where
|
||||
drvPath = ? and
|
||||
@@ -377,16 +377,16 @@ LocalStore::LocalStore(ref<const Config> config)
|
||||
state->stmts->QueryRealisedOutput.create(
|
||||
state->db,
|
||||
R"(
|
||||
select BuildTraceV2.id, Output.path, BuildTraceV2.signatures from BuildTraceV2
|
||||
inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath
|
||||
select Realisations.id, Output.path, Realisations.signatures from Realisations
|
||||
inner join ValidPaths as Output on Output.id = Realisations.outputPath
|
||||
where drvPath = ? and outputName = ?
|
||||
;
|
||||
)");
|
||||
state->stmts->QueryAllRealisedOutputs.create(
|
||||
state->db,
|
||||
R"(
|
||||
select outputName, Output.path from BuildTraceV2
|
||||
inner join ValidPaths as Output on Output.id = BuildTraceV2.outputPath
|
||||
select outputName, Output.path from Realisations
|
||||
inner join ValidPaths as Output on Output.id = Realisations.outputPath
|
||||
where drvPath = ?
|
||||
;
|
||||
)");
|
||||
@@ -604,7 +604,7 @@ void LocalStore::upgradeDBSchema(State & state)
|
||||
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
|
||||
doUpgrade(
|
||||
"20251016-ca-derivations",
|
||||
"20220326-ca-derivations",
|
||||
#include "ca-specific-schema.sql.gen.hh"
|
||||
);
|
||||
}
|
||||
@@ -648,8 +648,8 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
auto combinedSignatures = oldR->signatures;
|
||||
combinedSignatures.insert(info.signatures.begin(), info.signatures.end());
|
||||
state->stmts->UpdateRealisedOutput
|
||||
.use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(
|
||||
info.id.drvPath.to_string())(info.id.outputName)
|
||||
.use()(concatStringsSep(" ", Signature::toStrings(combinedSignatures)))(info.id.strHash())(
|
||||
info.id.outputName)
|
||||
.exec();
|
||||
} else {
|
||||
throw Error(
|
||||
@@ -663,7 +663,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
}
|
||||
} else {
|
||||
state->stmts->RegisterRealisedOutput
|
||||
.use()(info.id.drvPath.to_string())(info.id.outputName)(printStorePath(info.outPath))(
|
||||
.use()(info.id.strHash())(info.id.outputName)(printStorePath(info.outPath))(
|
||||
concatStringsSep(" ", Signature::toStrings(info.signatures)))
|
||||
.exec();
|
||||
}
|
||||
@@ -1539,7 +1539,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const std::set<Signa
|
||||
std::optional<std::pair<int64_t, UnkeyedRealisation>>
|
||||
LocalStore::queryRealisationCore_(LocalStore::State & state, const DrvOutput & id)
|
||||
{
|
||||
auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.drvPath.to_string())(id.outputName));
|
||||
auto useQueryRealisedOutput(state.stmts->QueryRealisedOutput.use()(id.strHash())(id.outputName));
|
||||
if (!useQueryRealisedOutput.next())
|
||||
return std::nullopt;
|
||||
auto realisationDbId = useQueryRealisedOutput.getInt(0);
|
||||
|
||||
@@ -242,15 +242,16 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
|
||||
|
||||
// If there are unknown output paths, attempt to find if the
|
||||
// paths are known to substituters through a realisation.
|
||||
auto outputHashes = staticOutputHashes(*this, *drv);
|
||||
knownOutputPaths = true;
|
||||
|
||||
for (auto & [outputName, _] : drv->outputs) {
|
||||
for (auto [outputName, hash] : outputHashes) {
|
||||
if (!bfd.outputs.contains(outputName))
|
||||
continue;
|
||||
|
||||
bool found = false;
|
||||
for (auto & sub : getDefaultSubstituters()) {
|
||||
auto realisation = sub->queryRealisation({drvPath, outputName});
|
||||
auto realisation = sub->queryRealisation({hash, outputName});
|
||||
if (!realisation)
|
||||
continue;
|
||||
found = true;
|
||||
@@ -367,7 +368,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd,
|
||||
OutputPathMap outputs;
|
||||
for (auto & [outputName, outputPathOpt] : outputsOpt) {
|
||||
if (!outputPathOpt)
|
||||
throw MissingRealisation(store, *bfd.drvPath, drvPath, outputName);
|
||||
throw MissingRealisation(bfd.drvPath->to_string(store), outputName);
|
||||
auto & outputPath = *outputPathOpt;
|
||||
outputs.insert_or_assign(outputName, outputPath);
|
||||
}
|
||||
@@ -391,7 +392,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
|
||||
bfd.output);
|
||||
auto & optPath = outputPaths.at(bfd.output);
|
||||
if (!optPath)
|
||||
throw MissingRealisation(store, *bfd.drvPath, drvPath, bfd.output);
|
||||
throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output);
|
||||
return *optPath;
|
||||
},
|
||||
},
|
||||
|
||||
@@ -42,18 +42,12 @@ create table if not exists NARs (
|
||||
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||
);
|
||||
|
||||
create table if not exists BuildTrace (
|
||||
create table if not exists Realisations (
|
||||
cache integer not null,
|
||||
|
||||
drvPath text not null,
|
||||
outputName text not null,
|
||||
|
||||
-- The following are null if the realisation is absent
|
||||
outputPath text,
|
||||
sigs text,
|
||||
|
||||
outputId text not null,
|
||||
content blob, -- Json serialisation of the realisation, or null if the realisation is absent
|
||||
timestamp integer not null,
|
||||
primary key (cache, drvPath, outputName),
|
||||
primary key (cache, outputId),
|
||||
foreign key (cache) references BinaryCaches(id) on delete cascade
|
||||
);
|
||||
|
||||
@@ -93,7 +87,7 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
|
||||
NarInfoDiskCacheImpl(
|
||||
const Settings & settings,
|
||||
SQLiteSettings sqliteSettings,
|
||||
Path dbPath = (getCacheDir() / "binary-cache-v8.sqlite").string())
|
||||
Path dbPath = (getCacheDir() / "binary-cache-v7.sqlite").string())
|
||||
: NarInfoDiskCache{settings}
|
||||
{
|
||||
auto state(_state.lock());
|
||||
@@ -129,24 +123,24 @@ struct NarInfoDiskCacheImpl : NarInfoDiskCache
|
||||
state->insertRealisation.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert or replace into BuildTrace(cache, drvPath, outputName, outputPath, sigs, timestamp)
|
||||
values (?, ?, ?, ?, ?, ?)
|
||||
insert or replace into Realisations(cache, outputId, content, timestamp)
|
||||
values (?, ?, ?, ?)
|
||||
)");
|
||||
|
||||
state->insertMissingRealisation.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert or replace into BuildTrace(cache, drvPath, outputName, timestamp)
|
||||
values (?, ?, ?, ?)
|
||||
insert or replace into Realisations(cache, outputId, timestamp)
|
||||
values (?, ?, ?)
|
||||
)");
|
||||
|
||||
state->queryRealisation.create(
|
||||
state->db,
|
||||
R"(
|
||||
select outputPath, sigs from BuildTrace
|
||||
where cache = ? and drvPath = ? and outputName = ? and
|
||||
((outputPath is null and timestamp > ?) or
|
||||
(outputPath is not null and timestamp > ?))
|
||||
select content from Realisations
|
||||
where cache = ? and outputId = ? and
|
||||
((content is null and timestamp > ?) or
|
||||
(content is not null and timestamp > ?))
|
||||
)");
|
||||
|
||||
/* Periodically purge expired entries from the database. */
|
||||
@@ -303,27 +297,22 @@ public:
|
||||
|
||||
auto now = time(0);
|
||||
|
||||
auto queryRealisation(state->queryRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(
|
||||
auto queryRealisation(state->queryRealisation.use()(cache.id)(id.to_string())(
|
||||
now - settings.ttlNegative)(now - settings.ttlPositive));
|
||||
|
||||
if (!queryRealisation.next())
|
||||
return {oUnknown, nullptr};
|
||||
return {oUnknown, 0};
|
||||
|
||||
if (queryRealisation.isNull(0))
|
||||
return {oInvalid, nullptr};
|
||||
return {oInvalid, 0};
|
||||
|
||||
try {
|
||||
return {
|
||||
oValid,
|
||||
std::make_shared<Realisation>(
|
||||
UnkeyedRealisation{
|
||||
.outPath = StorePath{queryRealisation.getStr(0)},
|
||||
.signatures = nlohmann::json::parse(queryRealisation.getStr(1)),
|
||||
},
|
||||
id),
|
||||
std::make_shared<Realisation>(nlohmann::json::parse(queryRealisation.getStr(0))),
|
||||
};
|
||||
} catch (Error & e) {
|
||||
e.addTrace({}, "reading build trace key-value from the local disk cache");
|
||||
e.addTrace({}, "while parsing the local disk cache");
|
||||
throw;
|
||||
}
|
||||
});
|
||||
@@ -369,9 +358,7 @@ public:
|
||||
auto & cache(getCache(*state, uri));
|
||||
|
||||
state->insertRealisation
|
||||
.use()(cache.id)(realisation.id.drvPath.to_string())(realisation.id.outputName)(
|
||||
realisation.outPath.to_string())(static_cast<nlohmann::json>(realisation.signatures).dump())(
|
||||
time(0))
|
||||
.use()(cache.id)(realisation.id.to_string())(static_cast<nlohmann::json>(realisation).dump())(time(0))
|
||||
.exec();
|
||||
});
|
||||
}
|
||||
@@ -382,7 +369,7 @@ public:
|
||||
auto state(_state.lock());
|
||||
|
||||
auto & cache(getCache(*state, uri));
|
||||
state->insertMissingRealisation.use()(cache.id)(id.drvPath.to_string())(id.outputName)(time(0)).exec();
|
||||
state->insertMissingRealisation.use()(cache.id)(id.to_string())(time(0)).exec();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -8,34 +8,28 @@ namespace nix {
|
||||
|
||||
MakeError(InvalidDerivationOutputId, Error);
|
||||
|
||||
DrvOutput DrvOutput::parse(const StoreDirConfig & store, std::string_view s)
|
||||
DrvOutput DrvOutput::parse(const std::string & strRep)
|
||||
{
|
||||
size_t n = s.rfind('^');
|
||||
if (n == s.npos)
|
||||
throw InvalidDerivationOutputId("Invalid derivation output id '%s': missing '^'", s);
|
||||
return DrvOutput{
|
||||
.drvPath = store.parseStorePath(s.substr(0, n)),
|
||||
.outputName = OutputName{s.substr(n + 1)},
|
||||
};
|
||||
}
|
||||
size_t n = strRep.find("!");
|
||||
if (n == strRep.npos)
|
||||
throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
|
||||
|
||||
std::string DrvOutput::render(const StoreDirConfig & store) const
|
||||
{
|
||||
return std::string(store.printStorePath(drvPath)) + "^" + outputName;
|
||||
return DrvOutput{
|
||||
.drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
|
||||
.outputName = strRep.substr(n + 1),
|
||||
};
|
||||
}
|
||||
|
||||
std::string DrvOutput::to_string() const
|
||||
{
|
||||
return std::string(drvPath.to_string()) + "^" + outputName;
|
||||
return strHash() + "!" + outputName;
|
||||
}
|
||||
|
||||
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
|
||||
{
|
||||
auto serialised = static_cast<nlohmann::json>(Realisation{*this, key});
|
||||
auto value = serialised.find("value");
|
||||
assert(value != serialised.end());
|
||||
value->erase("signatures");
|
||||
return serialised.dump();
|
||||
nlohmann::json serialized = Realisation{*this, key};
|
||||
serialized.erase("signatures");
|
||||
return serialized.dump();
|
||||
}
|
||||
|
||||
void UnkeyedRealisation::sign(const DrvOutput & key, const Signer & signer)
|
||||
@@ -67,20 +61,9 @@ const StorePath & RealisedPath::path() const &
|
||||
return std::visit([](auto & arg) -> auto & { return arg.getPath(); }, raw);
|
||||
}
|
||||
|
||||
MissingRealisation::MissingRealisation(
|
||||
const StoreDirConfig & store, const StorePath & drvPath, const OutputName & outputName)
|
||||
: Error("cannot operate on output '%s' of the unbuilt derivation '%s'", outputName, store.printStorePath(drvPath))
|
||||
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
|
||||
{
|
||||
}
|
||||
|
||||
MissingRealisation::MissingRealisation(
|
||||
const StoreDirConfig & store,
|
||||
const SingleDerivedPath & drvPath,
|
||||
const StorePath & drvPathResolved,
|
||||
const OutputName & outputName)
|
||||
: MissingRealisation{store, drvPathResolved, outputName}
|
||||
{
|
||||
addTrace({}, "looking up realisation for derivation '%s'", drvPath.to_string(store));
|
||||
return outPath == other.outPath;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -91,20 +74,12 @@ using namespace nix;
|
||||
|
||||
DrvOutput adl_serializer<DrvOutput>::from_json(const json & json)
|
||||
{
|
||||
auto obj = getObject(json);
|
||||
|
||||
return {
|
||||
.drvPath = valueAt(obj, "drvPath"),
|
||||
.outputName = getString(valueAt(obj, "outputName")),
|
||||
};
|
||||
return DrvOutput::parse(getString(json));
|
||||
}
|
||||
|
||||
void adl_serializer<DrvOutput>::to_json(json & json, const DrvOutput & drvOutput)
|
||||
{
|
||||
json = {
|
||||
{"drvPath", drvOutput.drvPath},
|
||||
{"outputName", drvOutput.outputName},
|
||||
};
|
||||
json = drvOutput.to_string();
|
||||
}
|
||||
|
||||
UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & json0)
|
||||
@@ -126,25 +101,25 @@ void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedReali
|
||||
json = {
|
||||
{"outPath", r.outPath},
|
||||
{"signatures", r.signatures},
|
||||
// back-compat
|
||||
{"dependentRealisations", json::object()},
|
||||
};
|
||||
}
|
||||
|
||||
Realisation adl_serializer<Realisation>::from_json(const json & json)
|
||||
Realisation adl_serializer<Realisation>::from_json(const json & json0)
|
||||
{
|
||||
auto obj = getObject(json);
|
||||
auto json = getObject(json0);
|
||||
|
||||
return {
|
||||
static_cast<UnkeyedRealisation>(valueAt(obj, "value")),
|
||||
static_cast<DrvOutput>(valueAt(obj, "key")),
|
||||
return Realisation{
|
||||
static_cast<UnkeyedRealisation>(json0),
|
||||
valueAt(json, "id"),
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<Realisation>::to_json(json & json, const Realisation & r)
|
||||
{
|
||||
json = {
|
||||
{"key", r.id},
|
||||
{"value", static_cast<const UnkeyedRealisation &>(r)},
|
||||
};
|
||||
json = static_cast<const UnkeyedRealisation &>(r);
|
||||
json["id"] = r.id;
|
||||
}
|
||||
|
||||
} // namespace nlohmann
|
||||
|
||||
@@ -503,7 +503,12 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << WorkerProto::Op::RegisterDrvOutput;
|
||||
WorkerProto::write(*this, *conn, info);
|
||||
if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
WorkerProto::write(*this, *conn, info.id);
|
||||
conn->to << std::string(info.outPath.to_string());
|
||||
} else {
|
||||
WorkerProto::write(*this, *conn, info);
|
||||
}
|
||||
conn.processStderr();
|
||||
}
|
||||
|
||||
@@ -513,10 +518,8 @@ void RemoteStore::queryRealisationUncached(
|
||||
try {
|
||||
auto conn(getConnection());
|
||||
|
||||
if (!conn->protoVersion.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
warn(
|
||||
"the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
if (conn->protoVersion.number < WorkerProto::Version::Number{1, 27}) {
|
||||
warn("the daemon is too old to support content-addressing derivations, please upgrade it to 2.4");
|
||||
return callback(nullptr);
|
||||
}
|
||||
|
||||
@@ -524,12 +527,21 @@ void RemoteStore::queryRealisationUncached(
|
||||
WorkerProto::write(*this, *conn, id);
|
||||
conn.processStderr();
|
||||
|
||||
callback([&]() -> std::shared_ptr<const UnkeyedRealisation> {
|
||||
auto realisation = WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(*this, *conn);
|
||||
if (!realisation)
|
||||
return nullptr;
|
||||
return std::make_shared<const UnkeyedRealisation>(*realisation);
|
||||
}());
|
||||
auto real = [&]() -> std::shared_ptr<const UnkeyedRealisation> {
|
||||
if (conn->protoVersion.number < WorkerProto::Version::Number{1, 31}) {
|
||||
auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read(*this, *conn);
|
||||
if (outPaths.empty())
|
||||
return nullptr;
|
||||
return std::make_shared<const UnkeyedRealisation>(UnkeyedRealisation{.outPath = *outPaths.begin()});
|
||||
} else {
|
||||
auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read(*this, *conn);
|
||||
if (realisations.empty())
|
||||
return nullptr;
|
||||
return std::make_shared<const UnkeyedRealisation>(*realisations.begin());
|
||||
}
|
||||
}();
|
||||
|
||||
callback(std::shared_ptr<const UnkeyedRealisation>(real));
|
||||
} catch (...) {
|
||||
return callback.rethrow();
|
||||
}
|
||||
@@ -611,19 +623,30 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
|
||||
|
||||
OutputPathMap outputs;
|
||||
auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath);
|
||||
auto drv = evalStore->readDerivation(drvPath);
|
||||
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
|
||||
auto built = resolveDerivedPath(*this, bfd, &*evalStore);
|
||||
for (auto & [output, outputPath] : built) {
|
||||
auto outputId = DrvOutput{drvPath, output};
|
||||
auto outputHash = get(outputHashes, output);
|
||||
if (!outputHash)
|
||||
throw Error(
|
||||
"the derivation '%s' doesn't have an output named '%s'",
|
||||
printStorePath(drvPath),
|
||||
output);
|
||||
auto outputId = DrvOutput{*outputHash, output};
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
|
||||
auto realisation = queryRealisation(outputId);
|
||||
if (!realisation)
|
||||
throw MissingRealisation(*this, outputId);
|
||||
success.builtOutputs.emplace(output, *realisation);
|
||||
throw MissingRealisation(outputId);
|
||||
success.builtOutputs.emplace(output, Realisation{*realisation, outputId});
|
||||
} else {
|
||||
success.builtOutputs.emplace(
|
||||
output,
|
||||
UnkeyedRealisation{
|
||||
.outPath = outputPath,
|
||||
Realisation{
|
||||
UnkeyedRealisation{
|
||||
.outPath = outputPath,
|
||||
},
|
||||
outputId,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -280,18 +280,9 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
|
||||
|
||||
for (auto & result : results) {
|
||||
if (auto * successP = result.tryGetSuccess()) {
|
||||
if (auto * pathBuilt = std::get_if<DerivedPathBuilt>(&result.path)) {
|
||||
// TODO ugly extra IO
|
||||
auto drvPath = resolveDerivedPath(*next, *pathBuilt->drvPath);
|
||||
for (auto & [outputName, output] : successP->builtOutputs) {
|
||||
newPaths.insert(output.outPath);
|
||||
newRealisations.insert(
|
||||
{output,
|
||||
{
|
||||
.drvPath = drvPath,
|
||||
.outputName = outputName,
|
||||
}});
|
||||
}
|
||||
for (auto & [outputName, output] : successP->builtOutputs) {
|
||||
newPaths.insert(output.outPath);
|
||||
newRealisations.insert(output);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
#include "nix/store/serve-protocol-impl.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/store/path-info.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@@ -29,19 +28,10 @@ BuildResult ServeProto::Serialise<BuildResult>::read(const StoreDirConfig & stor
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 3})
|
||||
conn.from >> res.timesBuilt >> isNonDeterministic >> res.startTime >> res.stopTime;
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 8}) {
|
||||
success.builtOutputs = ServeProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
|
||||
} else if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
for (auto & [output, realisation] : ServeProto::Serialise<StringMap>::read(store, conn)) {
|
||||
size_t n = output.find("!");
|
||||
if (n == output.npos)
|
||||
throw Error("Invalid derivation output id %s", output);
|
||||
success.builtOutputs.insert_or_assign(
|
||||
output.substr(n + 1),
|
||||
UnkeyedRealisation{
|
||||
StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}});
|
||||
}
|
||||
if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(store, conn);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation));
|
||||
}
|
||||
|
||||
res.inner = std::visit(
|
||||
@@ -73,18 +63,15 @@ void ServeProto::Serialise<BuildResult>::write(
|
||||
default value for the fields that don't exist in that case. */
|
||||
auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) {
|
||||
conn.to << errorMsg;
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 3})
|
||||
conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime;
|
||||
|
||||
if (conn.version >= ServeProto::Version{2, 8}) {
|
||||
ServeProto::write(store, conn, builtOutputs);
|
||||
} else if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
// We no longer support these types of realisations
|
||||
ServeProto::write(store, conn, StringMap{});
|
||||
if (conn.version >= ServeProto::Version{2, 6}) {
|
||||
DrvOutputs builtOutputsFullKey;
|
||||
for (auto & [output, realisation] : builtOutputs)
|
||||
builtOutputsFullKey.insert_or_assign(realisation.id, realisation);
|
||||
ServeProto::write(store, conn, builtOutputsFullKey);
|
||||
}
|
||||
};
|
||||
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const BuildResult::Failure & failure) {
|
||||
@@ -171,82 +158,4 @@ void ServeProto::Serialise<ServeProto::BuildOptions>::write(
|
||||
}
|
||||
}
|
||||
|
||||
UnkeyedRealisation ServeProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
|
||||
auto outPath = ServeProto::Serialise<StorePath>::read(store, conn);
|
||||
auto signatures = ServeProto::Serialise<std::set<Signature>>::read(store, conn);
|
||||
|
||||
return UnkeyedRealisation{
|
||||
.outPath = std::move(outPath),
|
||||
.signatures = std::move(signatures),
|
||||
};
|
||||
}
|
||||
|
||||
void ServeProto::Serialise<UnkeyedRealisation>::write(
|
||||
const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
ServeProto::write(store, conn, info.outPath);
|
||||
ServeProto::write(store, conn, info.signatures);
|
||||
}
|
||||
|
||||
DrvOutput ServeProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
|
||||
auto drvPath = ServeProto::Serialise<StorePath>::read(store, conn);
|
||||
auto outputName = ServeProto::Serialise<std::string>::read(store, conn);
|
||||
|
||||
return DrvOutput{
|
||||
.drvPath = std::move(drvPath),
|
||||
.outputName = std::move(outputName),
|
||||
};
|
||||
}
|
||||
|
||||
void ServeProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
|
||||
{
|
||||
if (conn.version < ServeProto::Version{2, 8}) {
|
||||
throw Error(
|
||||
"serve protocol %d.%d is too old (< 2.8) to support content-addressing derivations",
|
||||
conn.version.major,
|
||||
conn.version.minor);
|
||||
}
|
||||
ServeProto::write(store, conn, info.drvPath);
|
||||
ServeProto::write(store, conn, info.outputName);
|
||||
}
|
||||
|
||||
Realisation ServeProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
auto id = ServeProto::Serialise<DrvOutput>::read(store, conn);
|
||||
auto unkeyed = ServeProto::Serialise<UnkeyedRealisation>::read(store, conn);
|
||||
|
||||
return Realisation{
|
||||
std::move(unkeyed),
|
||||
std::move(id),
|
||||
};
|
||||
}
|
||||
|
||||
void ServeProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
|
||||
{
|
||||
ServeProto::write(store, conn, info.id);
|
||||
ServeProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -380,8 +380,9 @@ Store::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore
|
||||
return outputs;
|
||||
|
||||
auto drv = evalStore.readInvalidDerivation(path);
|
||||
for (auto & [outputName, _] : drv.outputs) {
|
||||
auto realisation = queryRealisation(DrvOutput{path, outputName});
|
||||
auto drvHashes = staticOutputHashes(*this, drv);
|
||||
for (auto & [outputName, hash] : drvHashes) {
|
||||
auto realisation = queryRealisation(DrvOutput{hash, outputName});
|
||||
if (realisation) {
|
||||
outputs.insert_or_assign(outputName, realisation->outPath);
|
||||
} else {
|
||||
@@ -401,7 +402,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * ev
|
||||
OutputPathMap result;
|
||||
for (auto & [outName, optOutPath] : resp) {
|
||||
if (!optOutPath)
|
||||
throw MissingRealisation(*this, path, outName);
|
||||
throw MissingRealisation(printStorePath(path), outName);
|
||||
result.insert_or_assign(outName, *optOutPath);
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -1958,10 +1958,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
|
||||
{
|
||||
.outPath = newInfo.path,
|
||||
},
|
||||
DrvOutput{
|
||||
.drvPath = drvPath,
|
||||
.outputName = outputName,
|
||||
},
|
||||
DrvOutput{oldinfo->outputHash, outputName},
|
||||
};
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().isImpure()) {
|
||||
store.signRealisation(thisRealisation);
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
#include "nix/store/worker-protocol-impl.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
#include "nix/store/path-info.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
|
||||
#include <chrono>
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -21,12 +20,6 @@ const WorkerProto::Version WorkerProto::latest = {
|
||||
.major = 1,
|
||||
.minor = 38,
|
||||
},
|
||||
.features =
|
||||
{
|
||||
std::string{
|
||||
WorkerProto::featureRealisationWithPath,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const WorkerProto::Version WorkerProto::minimum = {
|
||||
@@ -261,19 +254,10 @@ BuildResult WorkerProto::Serialise<BuildResult>::read(const StoreDirConfig & sto
|
||||
res.cpuUser = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
|
||||
res.cpuSystem = WorkerProto::Serialise<std::optional<std::chrono::microseconds>>::read(store, conn);
|
||||
}
|
||||
|
||||
if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
success.builtOutputs = WorkerProto::Serialise<std::map<OutputName, UnkeyedRealisation>>::read(store, conn);
|
||||
} else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
for (auto && [output, realisation] : WorkerProto::Serialise<StringMap>::read(store, conn)) {
|
||||
size_t n = output.find("!");
|
||||
if (n == output.npos)
|
||||
throw Error("Invalid derivation output id %s", output);
|
||||
success.builtOutputs.insert_or_assign(
|
||||
output.substr(n + 1),
|
||||
UnkeyedRealisation{
|
||||
StorePath{getString(valueAt(getObject(nlohmann::json::parse(realisation)), "outPath"))}});
|
||||
}
|
||||
if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation));
|
||||
}
|
||||
|
||||
res.inner = std::visit(
|
||||
@@ -312,15 +296,13 @@ void WorkerProto::Serialise<BuildResult>::write(
|
||||
WorkerProto::write(store, conn, res.cpuUser);
|
||||
WorkerProto::write(store, conn, res.cpuSystem);
|
||||
}
|
||||
|
||||
if (conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
WorkerProto::write(store, conn, builtOutputs);
|
||||
} else if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
// Don't support those types of realisations anymore.
|
||||
WorkerProto::write(store, conn, StringMap{});
|
||||
if (conn.version >= WorkerProto::Version{.number = {1, 28}}) {
|
||||
DrvOutputs builtOutputsFullKey;
|
||||
for (auto & [output, realisation] : builtOutputs)
|
||||
builtOutputsFullKey.insert_or_assign(realisation.id, realisation);
|
||||
WorkerProto::write(store, conn, builtOutputsFullKey);
|
||||
}
|
||||
};
|
||||
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const BuildResult::Failure & failure) {
|
||||
@@ -413,109 +395,4 @@ void WorkerProto::Serialise<WorkerProto::ClientHandshakeInfo>::write(
|
||||
}
|
||||
}
|
||||
|
||||
UnkeyedRealisation WorkerProto::Serialise<UnkeyedRealisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to understand build trace",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
|
||||
auto outPath = WorkerProto::Serialise<StorePath>::read(store, conn);
|
||||
auto signatures = WorkerProto::Serialise<std::set<Signature>>::read(store, conn);
|
||||
|
||||
return UnkeyedRealisation{
|
||||
.outPath = std::move(outPath),
|
||||
.signatures = std::move(signatures),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<UnkeyedRealisation>::write(
|
||||
const StoreDirConfig & store, WriteConn conn, const UnkeyedRealisation & info)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to understand build trace",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
WorkerProto::write(store, conn, info.outPath);
|
||||
WorkerProto::write(store, conn, info.signatures);
|
||||
}
|
||||
|
||||
std::optional<UnkeyedRealisation>
|
||||
WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
// Hack to improve compat
|
||||
(void) WorkerProto::Serialise<std::string>::read(store, conn);
|
||||
return std::nullopt;
|
||||
} else {
|
||||
auto temp = readNum<uint8_t>(conn.from);
|
||||
switch (temp) {
|
||||
case 0:
|
||||
return std::nullopt;
|
||||
case 1:
|
||||
return WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
|
||||
default:
|
||||
throw Error("Invalid optional build trace from remote");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<std::optional<UnkeyedRealisation>>::write(
|
||||
const StoreDirConfig & store, WriteConn conn, const std::optional<UnkeyedRealisation> & info)
|
||||
{
|
||||
if (!info) {
|
||||
conn.to << uint8_t{0};
|
||||
} else {
|
||||
conn.to << uint8_t{1};
|
||||
WorkerProto::write(store, conn, *info);
|
||||
}
|
||||
}
|
||||
|
||||
DrvOutput WorkerProto::Serialise<DrvOutput>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
|
||||
auto drvPath = WorkerProto::Serialise<StorePath>::read(store, conn);
|
||||
auto outputName = WorkerProto::Serialise<std::string>::read(store, conn);
|
||||
|
||||
return DrvOutput{
|
||||
.drvPath = std::move(drvPath),
|
||||
.outputName = std::move(outputName),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<DrvOutput>::write(const StoreDirConfig & store, WriteConn conn, const DrvOutput & info)
|
||||
{
|
||||
if (!conn.version.features.contains(WorkerProto::featureRealisationWithPath)) {
|
||||
throw Error(
|
||||
"the daemon is missing the '%s' protocol feature, needed to support content-addressing derivations",
|
||||
WorkerProto::featureRealisationWithPath);
|
||||
}
|
||||
WorkerProto::write(store, conn, info.drvPath);
|
||||
WorkerProto::write(store, conn, info.outputName);
|
||||
}
|
||||
|
||||
Realisation WorkerProto::Serialise<Realisation>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
auto id = WorkerProto::Serialise<DrvOutput>::read(store, conn);
|
||||
auto unkeyed = WorkerProto::Serialise<UnkeyedRealisation>::read(store, conn);
|
||||
|
||||
return Realisation{
|
||||
std::move(unkeyed),
|
||||
std::move(id),
|
||||
};
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<Realisation>::write(const StoreDirConfig & store, WriteConn conn, const Realisation & info)
|
||||
{
|
||||
WorkerProto::write(store, conn, info.id);
|
||||
WorkerProto::write(store, conn, static_cast<const UnkeyedRealisation &>(info));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
|
||||
@@ -346,11 +346,13 @@ static int main_build_remote(int argc, char ** argv)
|
||||
optResult = std::move(res[0]);
|
||||
}
|
||||
|
||||
auto outputHashes = staticOutputHashes(*store, drv);
|
||||
std::set<Realisation> missingRealisations;
|
||||
StorePathSet missingPaths;
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
|
||||
for (auto & outputName : wantedOutputs) {
|
||||
auto thisOutputId = DrvOutput{*drvPath, outputName};
|
||||
auto thisOutputHash = outputHashes.at(outputName);
|
||||
auto thisOutputId = DrvOutput{thisOutputHash, outputName};
|
||||
if (!store->queryRealisation(thisOutputId)) {
|
||||
debug("missing output %s", outputName);
|
||||
assert(optResult);
|
||||
@@ -360,7 +362,7 @@ static int main_build_remote(int argc, char ** argv)
|
||||
auto i = success.builtOutputs.find(outputName);
|
||||
assert(i != success.builtOutputs.end());
|
||||
auto & newRealisation = i->second;
|
||||
missingRealisations.insert({newRealisation, thisOutputId});
|
||||
missingRealisations.insert(newRealisation);
|
||||
missingPaths.insert(newRealisation.outPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,13 +163,10 @@ StoreWrapper::queryPathInfo(char * path, int base32)
|
||||
}
|
||||
|
||||
SV *
|
||||
StoreWrapper::queryRawRealisation(char * drvPath, char * outputName)
|
||||
StoreWrapper::queryRawRealisation(char * outputId)
|
||||
PPCODE:
|
||||
try {
|
||||
auto realisation = THIS->store->queryRealisation(DrvOutput{
|
||||
.drvPath = THIS->store->parseStorePath(drvPath),
|
||||
.outputName = outputName,
|
||||
});
|
||||
auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId));
|
||||
if (realisation)
|
||||
XPUSHs(sv_2mortal(newSVpv(static_cast<nlohmann::json>(*realisation).dump().c_str(), 0)));
|
||||
else
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
# shellcheck shell=bash
|
||||
source ../common.sh
|
||||
|
||||
# Need backend to support revamped CA
|
||||
requireDaemonNewerThan "2.34.0pre20251217"
|
||||
|
||||
enableFeatures "ca-derivations"
|
||||
|
||||
TODO_NixOS
|
||||
|
||||
@@ -49,14 +49,14 @@ fi
|
||||
clearStore
|
||||
nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore"
|
||||
clearStore
|
||||
rm -r "$REMOTE_STORE_DIR/build-trace-v2"
|
||||
rm -r "$REMOTE_STORE_DIR/realisations"
|
||||
nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
|
||||
# There's no easy way to check whether a realisation is present on the local
|
||||
# store − short of manually querying the db, but the build environment doesn't
|
||||
# have the sqlite binary − so we instead push things again, and check that the
|
||||
# realisations have correctly been pushed to the remote store
|
||||
nix copy --to "$REMOTE_STORE" --file ../simple.nix
|
||||
if [[ -z "$(ls "$REMOTE_STORE_DIR/build-trace-v2")" ]]; then
|
||||
if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
|
||||
echo "Realisations not rebuilt"
|
||||
exit 1
|
||||
fi
|
||||
@@ -71,5 +71,5 @@ buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
|
||||
# Try rebuilding, but remove the realisations from the remote cache to force
|
||||
# using the cachecache
|
||||
clearStore
|
||||
rm -r "$REMOTE_STORE_DIR"/build-trace-v2/*
|
||||
rm "$REMOTE_STORE_DIR"/realisations/*
|
||||
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
|
||||
|
||||
Reference in New Issue
Block a user