Print all port values

Question

I have this json file:

{ "data":  [    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test1", "{#PROC_ARGS}": "-l -c -g -k /etc/test1.conf", "{#PROC_PORT_1111}": "1111", "{#PROC_CONF}": "/etc/test1.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test2", "{#PROC_ARGS}": "-l -c -g -k /etc/test2.conf", "{#PROC_PORT_2222}": "2222", "{#PROC_PORT_3333}": "3333", "{#PROC_CONF}": "/etc/test2.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test3", "{#PROC_ARGS}": "-l -c -g -k /etc/test3.conf", "{#PROC_PORT_4444}": "4444", "{#PROC_CONF}": "/etc/test3.conf" }  ]}

This file is being read by this python script:

import json
import re
import sys
import unittest
import StringIO

def TestPorts(discoveryJson, spJson):
    jsn = json.load(discoveryJson)
    for dt in jsn['data']:
        try:
            id = dt['{#PROC_IDENT}']
            port = dt['{#PROC_PORT_1111}']
            spJson['data'].append({'{ID}': id, '{#PORT_1111}': port})
        except Exception as err:
            pass

def printTestPort(discFilespec, dumpDest=sys.stdout):
    portJson = {'data': []}
    try:
        with open(discFilespec) as discJson:
            TestPorts(discJson, portJson)
    except:
        pass
    json.dump(portJson, dumpDest)

if __name__ == '__main__':
    printTestPort('/tmp/file.json')

At the moment I can only print only one port value and id value in output:

{
    "data": [
        {
            "{#ID}": "test1",
            "{#PORT_1111}": "1111"
        }
    ]
}

How can I get the next output? :

{
        "data": [
            {
                "{#ID}": "test1",
                "{#PORT_1111}": "1111"
            },
            {
                "{#ID}": "test2",
                "{#PORT_2222}": "2222",
                "{#PORT_3333}": "3333"
            },
            {
                "{#ID}": "test3",
                "{#PORT_4444}": "4444"
            }
        ]
    }

Could you please help to achieve it?

---

Let me clarify one more time.

This json file may be changeable towards port values:

{ "data":  [    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test1", "{#PROC_ARGS}": "-l -c -g -k /etc/test1.conf", "{#PROC_PORT_1111}": "1111", "{#PROC_CONF}": "/etc/test1.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test2", "{#PROC_ARGS}": "-l -c -g -k /etc/test2.conf", "{#PROC_PORT_2222}": "2222", "{#PROC_PORT_3333}": "3333", "{#PROC_CONF}": "/etc/test2.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test3", "{#PROC_ARGS}": "-l -c -g -k /etc/test3.conf", "{#PROC_PORT_4444}": "4444", "{#PROC_CONF}": "/etc/test3.conf" }  ]}

So each of process instances there may have different amount of ports with different values. For example test1 may have 1237 7000 and 1234 port values test2 only 9004 and so on.

In my python code I was able to achieve only reading one of port values, but I don't know how to achieve in order it print all port values per process id.

For example:

{
        "data": [
            {
                "{#ID}": "test1",
                "{#PORT_1205}": "1205"
            },
            {
                "{#ID}": "test2",
                "{#PORT_442}": "442",
                "{#PORT_2004}": "2004"
            },
            {
                "{#ID}": "test3",
                "{#PORT_4444}": "9001"
            }
        ]
    }

So PORT values will automatically change in case of modifications of json file. Hope this time I explained more clearly.

Answer 1

Your original code threw a KeyError when the key '{#PROC_PORT_1111}' wasn't present so you couldn't capture other ports. Here is one way to do it - iterate over the items; check to see if you are interested in the item; massage it; keep it in a new container.

#setup
import json, io
from pprint import pprint
s = """{ "data":  [    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test1", "{#PROC_ARGS}": "-l -c -g -k /etc/test1.conf", "{#PROC_PORT_1111}": "1111", "{#PROC_CONF}": "/etc/test1.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test2", "{#PROC_ARGS}": "-l -c -g -k /etc/test2.conf", "{#PROC_PORT_2222}": "2222", "{#PROC_PORT_3333}": "3333", "{#PROC_CONF}": "/etc/test2.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test3", "{#PROC_ARGS}": "-l -c -g -k /etc/test3.conf", "{#PROC_PORT_4444}": "4444", "{#PROC_CONF}": "/etc/test3.conf" }  ]}"""
f = io.StringIO(s)

j = json.load(f)
new_j = {'data' : []}
for d in j['data']:
    new_d = {}
    new_d['{#ID}'] = d['{#PROC_IDENT}']
    for k, v in d.items():
        if k.startswith('{#PROC_PORT'):
            k = k.replace('PROC_', '')
            new_d[k] = v
    new_j['data'].append(new_d)


>>> pprint(new_j)

{'data': [{'{#ID}': 'test1', '{#PORT_1111}': '1111'},
          {'{#ID}': 'test2', '{#PORT_2222}': '2222', '{#PORT_3333}': '3333'},
          {'{#ID}': 'test3', '{#PORT_4444}': '4444'}]}
>>>

---

Using the regular expressions. I'm using the regex module because it saves repeat captures which is needed for processes with multiple ports

import json
import regex
from pprint import pprint

pattern = r'{.*?(?P<id>"{#PROC_IDENT}"[^,]+).*?((?P<ports>"{#PROC_PORT_\d+}"[^,]+),\s?)+'
r = regex.compile(pattern)
# formatting string
new_json = """{{ "data": [{}  ]}}"""
items = []

s = """{ "data":  [    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test1", "{#PROC_ARGS}": "-l -c -g -k /etc/test1.conf", "{#PROC_PORT_1111}": "1111", "{#PROC_CONF}": "/etc/test1.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test2", "{#PROC_ARGS}": "-l -c -g -k /etc/test2.conf", "{#PROC_PORT_2222}": "2222", "{#PROC_PORT_3333}": "3333", "{#PROC_CONF}": "/etc/test2.conf" },    { "{#PROC}": "/usr/local/test", "{#PROC_IDENT}": "test3", "{#PROC_ARGS}": "-l -c -g -k /etc/test3.conf", "{#PROC_PORT_4444}": "4444", "{#PROC_CONF}": "/etc/test3.conf" }  ]}"""
f = io.StringIO(s)
data = f.read()

#with open('s.txt') as f:
#    data = f.read()

for m in r.finditer(data):
    d = m.capturesdict()
    d['id'][0] = d['id'][0].replace('PROC_IDENT', 'ID')
    d['ports'] = [port.replace('PROC_', '') for port in d['ports']]
    s = ','.join(thing for v in d.values() for thing in v)
    items.append('{{{}}}'.format(s))

new_json = new_json.format(',    '.join(items))
j = json.loads(new_json)


>>> pprint(j)
{'data': [{'{#ID}': 'test1', '{#PORT_1111}': '1111'},
          {'{#ID}': 'test2', '{#PORT_2222}': '2222', '{#PORT_3333}': '3333'},
          {'{#ID}': 'test3', '{#PORT_4444}': '4444'}]}
>>>

Answer 2

To the best of my knowledge, I understood that your field "{#PROC_PORT_2222}" changes with numbers, i.e. "{#PROC_PORT_XXXX}" so in this case we need to use regex to match any string which has "{#PROC_PORT_}" as fixed string.

import re
import json

with open('s.txt') as data_file:
    data = json.load(data_file)

k = data['data']
regex = r"{#PROC_PORT_[0-9]{4}}"

test_str = str(k)
lst=[]
matches = re.finditer(regex, test_str)
for matchNum, match in enumerate(matches):
    matchNum = matchNum + 1
    lst.append("{match}".format(match=match.group()))


for b in k:
    for a in lst:
        try:
            print b[str(a)]
        except:
            pass

Where s.txt is the txt file having json.

This gives the output.

1111
3333
2222
4444

P.S. If you meant the key name is just PORT not PROC_PORT, then replace the line

regex = r"{#PROC_PORT_[0-9]{4}}"

by regex = r"{#PORT_[0-9]{4}}"

P.P.S I assumed that the numbers which will be changing will be 4 digit, if not, then please comment below

Answer 3

You need to update the {#PROC_PORT_1111} key in your loop so that it gets the next port (2222, 3333, 4444, etc.) on every iteration. I have added a incr variable to keep track of that. Also edit your function to use get whenever you are accessing the dictionary:

def TestPorts(discoveryJson, spJson):
    jsn = json.load(discoveryJson)
    incr = 1111;
    for dt in jsn.get('data'):
        try:
            id = dt.get('{#PROC_IDENT}')
            port = dt.get('{#PROC_PORT_' + str(incr) + '}')
            spJson.get('data').append({'{ID}': id, '{#PORT_' + str(incr) + '}': port})
            incr += incr;
        except Exception as err:
            pass

If you put a print statement in the except branch, you'll notice that your execution will hit that branch twice due to a KeyError. It's usually a better practice to use get rather than [] since the former never throws a KeyError whereas the latter does.

Resource: dict.get