Reputation: 623
overwrite a json file with objects from another json file using jq
MODIFIED:
Hello I'm having problems on merging 2 files basically I have 2 json files with this structure:
[
{
"uri": "some/url.feature",
"id": "safety-tests",
"keyword": "Feature",
"name": "Safety Tests",
"description": "Some description",
"line": 2,
"tags": [
{
"name": "@sometag",
"line": 1
}
],
"elements": [
{
"id": "some-element-id",
"keyword": "Scenario Outline",
"name": ": Some scenario name",
"description": "",
"line": 46,
"type": "scenario",
"tags": [
{
"name": "@sometag",
"line": 1
},
{
"name": "@someothertag",
"line": 31
}
],
"before": [
{
"match": {
"location": "some/test/file.rb:201"
},
"result": {
"status": "passed",
"duration": 15000
}
},
{
"match": {
"location": "some/other/file.rb:5"
},
"result": {
"status": "passed",
"duration": 1722192000
}
}
],
"steps": [
{
"keyword": "Given ",
"name": "Some step name",
"line": 46,
"output": [
"Some output"
],
"match": {
"location": "some/other/path/to/other/file.rb:137"
},
"result": {
"status": "passed",
"duration": 989158000
}
},
{
"keyword": "When ",
"name": "some other step",
"line": 46,
"output": [
"WARNING: static wait for 1 seconds."
],
"match": {
"location": "some/other/path/to/other/file.rb:80"
},
"result": {
"status": "passed",
"duration": 2700052000
}
},
{
"keyword": "And ",
"name": "Some other name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:38"
},
"result": {
"status": "passed",
"duration": 954225000
}
},
{
"keyword": "Then ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 38792000
}
},
{
"keyword": "And ",
"name": "And again some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 39268000
}
},
{
"keyword": "And ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 55637000
}
},
{
"keyword": "And ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 38375000
}
},
{
"keyword": "When ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:12"
},
"result": {
"status": "passed",
"duration": 751416000
}
},
{
"keyword": "And ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 28043000
}
},
{
"keyword": "Then ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:20"
},
"result": {
"status": "passed",
"duration": 5204000
}
}
],
"after": [
{
"match": {
"location": "some/other/path/to/other/file.rb:91"
},
"result": {
"status": "passed",
"duration": 20000
}
},
{
"match": {
"location": "some/other/path/to/other/file.rb:52"
},
"result": {
"status": "passed",
"duration": 5585000
}
},
{
"match": {
"location": "some/other/path/to/other/file.rb:27"
},
"result": {
"status": "passed",
"duration": 168146000
}
},
{
"match": {
"location": "some/other/path/to/other/file.rb:428"
},
"result": {
"status": "passed",
"duration": 62000
}
}
]
},
{
"id": "some-element-id",
"keyword": "Scenario Outline",
"name": ": Some scenario name",
"description": "",
"line": 46,
"type": "scenario",
"tags": [
{
"name": "@sometag",
"line": 1
},
{
"name": "@someothertag",
"line": 31
}
],
"before": [
{
"match": {
"location": "some/test/file.rb:201"
},
"result": {
"status": "passed",
"duration": 15000
}
},
{
"match": {
"location": "some/other/file.rb:5"
},
"result": {
"status": "passed",
"duration": 1722192000
}
}
],
"steps": [
{
"keyword": "Given ",
"name": "Some step name",
"line": 46,
"output": [
"Some output"
],
"match": {
"location": "some/other/path/to/other/file.rb:137"
},
"result": {
"status": "passed",
"duration": 989158000
}
},
{
"keyword": "When ",
"name": "some other step",
"line": 46,
"output": [
"WARNING: static wait for 1 seconds."
],
"match": {
"location": "some/other/path/to/other/file.rb:80"
},
"result": {
"status": "passed",
"duration": 2700052000
}
},
{
"keyword": "And ",
"name": "Some other name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:38"
},
"result": {
"status": "passed",
"duration": 954225000
}
},
{
"keyword": "Then ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 38792000
}
},
{
"keyword": "And ",
"name": "And again some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 39268000
}
},
{
"keyword": "And ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 55637000
}
},
{
"keyword": "And ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 38375000
}
},
{
"keyword": "When ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:12"
},
"result": {
"status": "passed",
"duration": 751416000
}
},
{
"keyword": "And ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:2"
},
"result": {
"status": "passed",
"duration": 28043000
}
},
{
"keyword": "Then ",
"name": "Some other step name",
"line": 46,
"match": {
"location": "some/other/path/to/other/file.rb:20"
},
"result": {
"status": "passed",
"duration": 5204000
}
}
],
"after": [
{
"match": {
"location": "some/other/path/to/other/file.rb:91"
},
"result": {
"status": "passed",
"duration": 20000
}
},
{
"match": {
"location": "some/other/path/to/other/file.rb:52"
},
"result": {
"status": "passed",
"duration": 5585000
}
},
{
"match": {
"location": "some/other/path/to/other/file.rb:27"
},
"result": {
"status": "passed",
"duration": 168146000
}
},
{
"match": {
"location": "some/other/path/to/other/file.rb:428"
},
"result": {
"status": "passed",
"duration": 62000
}
}
]
}
]
}
]
where elements can contain any number of objects on either file. These are test results from cucumber so usually file A contains more elements than file B since file B is a re-run of the failed tests in file A.
For example. If on the first pass we ran 100 tests, file A elements array will contain 100 objects of with the format described above. However if from those 100 tests 50 of them failed, file B elements array will contain 50 objects. What I want to do is to overwrite file A elements array with file B's just adding the elements that repeat in both. Something like
if file A had
"elements":[{a:1, b:2, c:3, d:2, e:9, f:4}]
and file B had
"elements":[{d:5}]
I'd like the new file to have
"elements":[{a:1, b:2, c:3, d:5, e:9, f:4}]
So far I've got
jq '.[].elements' path/to/file/b > path/to/new/file
jq --argfile file path/to/new/file '.[].elements += $file' path/to/file/b
That puts together whatever file B contains in the elements array within the elements array in file A but doesn't remove the duplicated object inside of it.
I tried to use unique but no clue on how to use it. Any ideas?
After a few responses here I got
jq --argfile b ~/Desktop/cucumber-rerun.json '.[0].elements[4] *= $b[0].elements[0]' ~/Desktop/cucumber.json
to works since, in my actual example, I knew the element 4 in file A is the one I want to overwrite with the 1 and only element in file B. However that doesn't work for me since both files are autogenerated and the order of the objects is unknown.
I'd like to have a command that sees both files compared them and autodetects repeated objects from A and B and overwrite those in A with those in B
Upvotes: 1
Views: 5240
Answers (2)
Reputation: 14715
Here is a solution which uses Object Multiplication. Assuming your data is in A.json and B.json:
$ jq -M --argfile b B.json '.[0].elements[0] *= $b[0].elements[0]' A.json
produces
[
{
"uri": "https://someurl.com",
"id": "some-id",
"keyword": "SomeKeyword",
"name": "Some Name",
"description": "Some description for that test result",
"line": 2,
"tags": [
{
"name": "@sometag",
"line": 1
}
],
"elements": [
{
"a": 5,
"b": 2
}
]
}
]
This approach is easily generalized if your arrays contain more data but you'll need to understand how corresponding elements should be identified.
Regarding the revised question, here is a filter which updates objects of A.json with corresponding objects of B.json having the same .id:
def INDEX(stream; idx_expr):
reduce stream as $row ({};
.[$row|idx_expr| if type != "string" then tojson else . end] |= $row);
def merge_by_id(a;b):
if b then INDEX(a[];.id) * INDEX(b[];.id) | map(.) else a end;
INDEX($b[];.id) as $i
| map( .elements = merge_by_id(.elements; $i[.id].elements) )
For example if the above filter is in filter.jq, A.json contains the revised sample data and B.json contains
[
{
"id": "safety-tests",
"elements": [
{
"id": "some-element-id",
"description": "updated description"
}
]
}
]
The command
$ jq -M --argfile b B.json -f filter.jq A.json
generates the result
[
{
"uri": "some/url.feature",
"id": "safety-tests", <------ top level .id
...
"elements": [
{
"id": "some-element-id", <------ element .id
"keyword": "Scenario Outline",
"name": ": Some scenario name",
"description": "updated description", <------ updated value
"line": 46,
"type": "scenario",
...
Note that the above solution assumes the .id of the elements in A.json are unique otherwise merge_by_id won't produce the desired output. In that case the following filter should suffice:
def INDEX(stream; idx_expr):
reduce stream as $row ({};
.[$row|idx_expr| if type != "string" then tojson else . end] |= $row);
(INDEX($b[];.id) | map_values(INDEX(.elements[];.id))) as $i
| map( $i[.id] as $o | if $o then .elements |= map($o[.id]//.) else . end )
This filter only requires the .id of the objects in B.json to be unique. If it's possible for there to be non-unique elements in both A.json and B.json then a more sophisticated mapping then this one will be required.
Here is a version of the filter with comments:
def INDEX(stream; idx_expr):
reduce stream as $row ({};
.[$row|idx_expr| if type != "string" then tojson else . end] |= $row);
# first create a lookup table for elements from B.json
( # [{id:x, elements:[{id:y, ...}]}]
INDEX($b[];.id) # -> {x: {id:x, elements:[{id:y, ...}]}..}
| map_values(INDEX(.elements[];.id)) # -> {x: {y: {id:y, ...}}}
) as $i
# update A.json objects
| map( # for each object in A.json
$i[.id] as $o # do we have updated values from B.json ?
| if $o then .elements |= map($o[.id]//.) # if so then replace corresponding elements
else . end # otherwise leave object unchanged
)
Upvotes: 1
Reputation: 116967
There are two key insights required to solve your problem concisely:
if A and B are two objects, then you can combine them with priority being given to B by writing: A + B
To update an object "in place", we can use |= or += as appropriate.
In your case, we can apply these insights by writing:
.[0].elements[0] += $B[0].elements[0]
assuming an invocation along the following lines:
jq --argfile B B.json -f combine.jq A.json
Output
With your input, the output includes:
"elements": [
{
"a": 5,
"b": 2
}
]
As per the requirements.
Further notes
To overwrite A.json with the output produced by the above invocation of jq, you might want to use sponge, but please be aware of the risks.
If you don't want to use --argfile for whatever reason, you could use --slurpfile instead, but then you'd have to write $B[0][0]
Upvotes: 0
Related Questions
- Merge 2 JSON objects from 2 files using jq
- How to merge 2 JSON objects from 2 files using jq?
- Merging two JSON file and reconstructor the JSON by jq
- jq: add new elements to list from other file
- How to merge two json files with jq
- JQ to merge new values between 2 json files
- Merging two files using jq
- Update one JSON file values with values from another JSON using JQ (on all levels)
- Update one JSON file values with values from another JSON using JQ
- Update keypairs in one json file with another json using jq (at another level)