CODEWITHSUNDEEP
javascriptpythonpython-3.xdata-structuresscrapy
Jinnrry
Jinnrry

Reputation: 455

How can I parse this non-JSON data?

This is my data. It is very similar to JSON, but it's not. Now I don't know how to use it in Python 3.

(["icolor_ex": 0, "iDod_All": 91, "commu_gid": 0, "normal_horse": 0, "iBeastSki1": 0, "rent": 0, "datang_feat": 452, "total_avatar": 0, "iDef_All": 188, "iMag_All": 74, "iBadness": 0, "iLearnCash": 13205, "bid": 0, "iRes_All": 81, "iMp": 339, "iSchOffer": 0, "iSewski": 0, "more_attr": (["attrs": ({
    (["lv": 0, "idx": 1, ]),
    (["lv": 0, "idx": 2, ]),
    (["lv": 0, "idx": 8, ]),
    (["lv": 0, "idx": 12, ]),
    (["lv": 0, "idx": 7, ]),
    (["lv": 0, "idx": 6, ]),
    (["lv": 0, "idx": 11, ]),
    (["lv": 0, "idx": 4, ]),
    (["lv": 0, "idx": 5, ]),
    (["lv": 0, "idx": 3, ]),
    (["lv": 0, "idx": 9, ]),
    (["lv": 0, "idx": 10, ]),
    (["lv": 91, "idx": 13, ]),
    (["lv": 155, "idx": 14, ]),
}), ]), "HugeHorse": ([]), "iCGBoxAmount": 0, "iSumAmount": 3, "sword_score": 0, "all_skills": (["179": 1, "30": 45, "29": 45, "52032": 1, "34": 45, "25": 1, "52016": 1, "31": 45, "32": 45, "196": 1, "33": 1, "52031": 1, ]), "iPoint": 350, "iNutsNum": 0, "iOrgOffer": 0, "iMarry2": 0, "iExptSki2": 0, "sum_exp": 0, "iTotalMagDam_all": 155, "ExpJwBase": 1000000000, "iTotalMagDef_all": 155, "iSumAmountEx": 0, "idbid_desc": ({}), "cName": "Heler", "AllEquip": ([]), "iBeastSki4": 0, "AllSummon": ({}), "iExptSki4": 0, "iMaxExpt2": 20, "HeroScore": 0, "shenqi_pos": ({
    0,
    0,
}), "cOrg": "", "iPride": 733, "iHp_Max": 464, "iSpe_All": 79, "fabao": ([]), "iDamage_All": 107, "iMarry": 0, "outdoor_level": 0, "iCash": 32029, "shenqi": ([]), "iSkiPoint": 0, "iDex_All": 79, "iBeastSki2": 0, "usernum": 20759973, "iIcon": 9, "addPoint": 0, "commu_name": 0, "iDesc": 0, "iSchool": 7, "iGoodness": 0, "igoodness_sav": 0, "iStr_All": 80, "AchPointTotal": 91, "iCor_All": 81, "jiyuan": 0, "iCGBodyAmount": 0, "iAtt_All": 166, "iZhuanZhi": 0, "total_horse": 0, "iMagDef_All": 155, "iCGTotalAmount": 0, "iMp_Max": 339, "iSmithski": 0, "iPcktPage": 0, "iRace": 3, "TA_iAllPoint": 0, "iExptSki1": 0, "ori_desc": 146, "ExpJw": 0, "ExAvt": ([]), "i3FlyLv": 0, "rent_level": 0, "ori_race": 3, "iMaxExpt1": 20, "TA_iAllNewPoint": 0, "iBeastSki3": 0, "changesch": ({}), "AllRider": ([]), "iErrantry": 0, "iHp": 454, "iGrade": 69, "pet": ({}), "iMaxExpt3": 20, "iSaving": 0, "shenqi_yellow": "", "xianyu": 0, "iHp_Eff": 464, "energy": 0, "iMaxExpt4": 20, "iExptSki3": 0, "iUpExp": 1084477, "propKept": ([]), "farm_level": 0, "iExptSki5": 0, ])

Upvotes: 2

Views: 1388

Answers (3)

PM 2Ring
PM 2Ring

Reputation: 55489

If the brackets {}[]() are only used for grouping and never appear inside any of the key or value strings, then we can convert that data into a form that can be parsed by ast.literal_eval. We just need to swap the square brackets and the braces, and remove the parentheses. That's easily done using str.translate to perform the translation and str.maketrans to build the translation table. This is faster than using str.replace because it can be done in one step.

Once the string data has been translated, we convert it to a Python object using ast.literal_eval. I'll use json.dumps to convert that Python object to JSON, just so I can print it nicely.

from ast import literal_eval
import json

data = '''\
(["icolor_ex": 0, "iDod_All": 91, "commu_gid": 0, "normal_horse": 0,
"iBeastSki1": 0, "rent": 0, "datang_feat": 452, "total_avatar": 0,
"iDef_All": 188, "iMag_All": 74, "iBadness": 0, "iLearnCash": 13205,
"bid": 0, "iRes_All": 81, "iMp": 339, "iSchOffer": 0, "iSewski": 0,
"more_attr": (["attrs": ({
    (["lv": 0, "idx": 1, ]),
    (["lv": 0, "idx": 2, ]),
    (["lv": 0, "idx": 8, ]),
    (["lv": 0, "idx": 12, ]),
    (["lv": 0, "idx": 7, ]),
    (["lv": 0, "idx": 6, ]),
    (["lv": 0, "idx": 11, ]),
    (["lv": 0, "idx": 4, ]),
    (["lv": 0, "idx": 5, ]),
    (["lv": 0, "idx": 3, ]),
    (["lv": 0, "idx": 9, ]),
    (["lv": 0, "idx": 10, ]),
    (["lv": 91, "idx": 13, ]),
    (["lv": 155, "idx": 14, ]),
}), ]), "HugeHorse": ([]), "iCGBoxAmount": 0, "iSumAmount": 3,
"sword_score": 0, "all_skills": (["179": 1, "30": 45, "29": 45, "52032":
1, "34": 45, "25": 1, "52016": 1, "31": 45, "32": 45, "196": 1, "33": 1,
"52031": 1, ]), "iPoint": 350, "iNutsNum": 0, "iOrgOffer": 0, "iMarry2":
0, "iExptSki2": 0, "sum_exp": 0, "iTotalMagDam_all": 155, "ExpJwBase":
1000000000, "iTotalMagDef_all": 155, "iSumAmountEx": 0, "idbid_desc":
({}), "cName": "Heler", "AllEquip": ([]), "iBeastSki4": 0, "AllSummon":
({}), "iExptSki4": 0, "iMaxExpt2": 20, "HeroScore": 0, "shenqi_pos": ({
    0,
    0,
}), "cOrg": "", "iPride": 733, "iHp_Max": 464, "iSpe_All": 79, "fabao":
([]), "iDamage_All": 107, "iMarry": 0, "outdoor_level": 0, "iCash":
32029, "shenqi": ([]), "iSkiPoint": 0, "iDex_All": 79, "iBeastSki2": 0,
"usernum": 20759973, "iIcon": 9, "addPoint": 0, "commu_name": 0,
"iDesc": 0, "iSchool": 7, "iGoodness": 0, "igoodness_sav": 0,
"iStr_All": 80, "AchPointTotal": 91, "iCor_All": 81, "jiyuan": 0,
"iCGBodyAmount": 0, "iAtt_All": 166, "iZhuanZhi": 0, "total_horse": 0,
"iMagDef_All": 155, "iCGTotalAmount": 0, "iMp_Max": 339, "iSmithski": 0,
"iPcktPage": 0, "iRace": 3, "TA_iAllPoint": 0, "iExptSki1": 0,
"ori_desc": 146, "ExpJw": 0, "ExAvt": ([]), "i3FlyLv": 0, "rent_level":
0, "ori_race": 3, "iMaxExpt1": 20, "TA_iAllNewPoint": 0, "iBeastSki3":
0, "changesch": ({}), "AllRider": ([]), "iErrantry": 0, "iHp": 454,
"iGrade": 69, "pet": ({}), "iMaxExpt3": 20, "iSaving": 0,
"shenqi_yellow": "", "xianyu": 0, "iHp_Eff": 464, "energy": 0,
"iMaxExpt4": 20, "iExptSki3": 0, "iUpExp": 1084477, "propKept": ([]),
"farm_level": 0, "iExptSki5": 0, ])
'''

new_data = data.translate(str.maketrans('{}[]', '[]{}', '()'))
obj = literal_eval(new_data)
print(json.dumps(obj, indent=4))

output

{
    "icolor_ex": 0,
    "iDod_All": 91,
    "commu_gid": 0,
    "normal_horse": 0,
    "iBeastSki1": 0,
    "rent": 0,
    "datang_feat": 452,
    "total_avatar": 0,
    "iDef_All": 188,
    "iMag_All": 74,
    "iBadness": 0,
    "iLearnCash": 13205,
    "bid": 0,
    "iRes_All": 81,
    "iMp": 339,
    "iSchOffer": 0,
    "iSewski": 0,
    "more_attr": {
        "attrs": [
            {
                "lv": 0,
                "idx": 1
            },
            {
                "lv": 0,
                "idx": 2
            },
            {
                "lv": 0,
                "idx": 8
            },
            {
                "lv": 0,
                "idx": 12
            },
            {
                "lv": 0,
                "idx": 7
            },
            {
                "lv": 0,
                "idx": 6
            },
            {
                "lv": 0,
                "idx": 11
            },
            {
                "lv": 0,
                "idx": 4
            },
            {
                "lv": 0,
                "idx": 5
            },
            {
                "lv": 0,
                "idx": 3
            },
            {
                "lv": 0,
                "idx": 9
            },
            {
                "lv": 0,
                "idx": 10
            },
            {
                "lv": 91,
                "idx": 13
            },
            {
                "lv": 155,
                "idx": 14
            }
        ]
    },
    "HugeHorse": {},
    "iCGBoxAmount": 0,
    "iSumAmount": 3,
    "sword_score": 0,
    "all_skills": {
        "179": 1,
        "30": 45,
        "29": 45,
        "52032": 1,
        "34": 45,
        "25": 1,
        "52016": 1,
        "31": 45,
        "32": 45,
        "196": 1,
        "33": 1,
        "52031": 1
    },
    "iPoint": 350,
    "iNutsNum": 0,
    "iOrgOffer": 0,
    "iMarry2": 0,
    "iExptSki2": 0,
    "sum_exp": 0,
    "iTotalMagDam_all": 155,
    "ExpJwBase": 1000000000,
    "iTotalMagDef_all": 155,
    "iSumAmountEx": 0,
    "idbid_desc": [],
    "cName": "Heler",
    "AllEquip": {},
    "iBeastSki4": 0,
    "AllSummon": [],
    "iExptSki4": 0,
    "iMaxExpt2": 20,
    "HeroScore": 0,
    "shenqi_pos": [
        0,
        0
    ],
    "cOrg": "",
    "iPride": 733,
    "iHp_Max": 464,
    "iSpe_All": 79,
    "fabao": {},
    "iDamage_All": 107,
    "iMarry": 0,
    "outdoor_level": 0,
    "iCash": 32029,
    "shenqi": {},
    "iSkiPoint": 0,
    "iDex_All": 79,
    "iBeastSki2": 0,
    "usernum": 20759973,
    "iIcon": 9,
    "addPoint": 0,
    "commu_name": 0,
    "iDesc": 0,
    "iSchool": 7,
    "iGoodness": 0,
    "igoodness_sav": 0,
    "iStr_All": 80,
    "AchPointTotal": 91,
    "iCor_All": 81,
    "jiyuan": 0,
    "iCGBodyAmount": 0,
    "iAtt_All": 166,
    "iZhuanZhi": 0,
    "total_horse": 0,
    "iMagDef_All": 155,
    "iCGTotalAmount": 0,
    "iMp_Max": 339,
    "iSmithski": 0,
    "iPcktPage": 0,
    "iRace": 3,
    "TA_iAllPoint": 0,
    "iExptSki1": 0,
    "ori_desc": 146,
    "ExpJw": 0,
    "ExAvt": {},
    "i3FlyLv": 0,
    "rent_level": 0,
    "ori_race": 3,
    "iMaxExpt1": 20,
    "TA_iAllNewPoint": 0,
    "iBeastSki3": 0,
    "changesch": [],
    "AllRider": {},
    "iErrantry": 0,
    "iHp": 454,
    "iGrade": 69,
    "pet": [],
    "iMaxExpt3": 20,
    "iSaving": 0,
    "shenqi_yellow": "",
    "xianyu": 0,
    "iHp_Eff": 464,
    "energy": 0,
    "iMaxExpt4": 20,
    "iExptSki3": 0,
    "iUpExp": 1084477,
    "propKept": {},
    "farm_level": 0,
    "iExptSki5": 0
}

If the brackets {}[]() can appear inside the key or value strings, then you cannot use simple techniques like this, you need to use a proper parser. You could write a parser from scratch, or you could construct one using a 3rd-party library, eg pyparsing.

Upvotes: 1

Jaromanda X
Jaromanda X

Reputation: 1

I don't know how you'd do it in Python - but since you tagged the question javascript, what you can do is get that string and perform the following on it

var failedJson = `(["icolor_ex": 0, "iDod_All": 91, "commu_gid": 0, "normal_horse": 0, "iBeastSki1": 0, "rent": 0, "datang_feat": 452, "total_avatar": 0, "iDef_All": 188, "iMag_All": 74, "iBadness": 0, "iLearnCash": 13205, "bid": 0, "iRes_All": 81, "iMp": 339, "iSchOffer": 0, "iSewski": 0, "more_attr": (["attrs": ({
    (["lv": 0, "idx": 1, ]),
    (["lv": 0, "idx": 2, ]),
    (["lv": 0, "idx": 8, ]),
    (["lv": 0, "idx": 12, ]),
    (["lv": 0, "idx": 7, ]),
    (["lv": 0, "idx": 6, ]),
    (["lv": 0, "idx": 11, ]),
    (["lv": 0, "idx": 4, ]),
    (["lv": 0, "idx": 5, ]),
    (["lv": 0, "idx": 3, ]),
    (["lv": 0, "idx": 9, ]),
    (["lv": 0, "idx": 10, ]),
    (["lv": 91, "idx": 13, ]),
    (["lv": 155, "idx": 14, ]),
}), ]), "HugeHorse": ([]), "iCGBoxAmount": 0, "iSumAmount": 3, "sword_score": 0, "all_skills": (["179": 1, "30": 45, "29": 45, "52032": 1, "34": 45, "25": 1, "52016": 1, "31": 45, "32": 45, "196": 1, "33": 1, "52031": 1, ]), "iPoint": 350, "iNutsNum": 0, "iOrgOffer": 0, "iMarry2": 0, "iExptSki2": 0, "sum_exp": 0, "iTotalMagDam_all": 155, "ExpJwBase": 1000000000, "iTotalMagDef_all": 155, "iSumAmountEx": 0, "idbid_desc": ({}), "cName": "Heler", "AllEquip": ([]), "iBeastSki4": 0, "AllSummon": ({}), "iExptSki4": 0, "iMaxExpt2": 20, "HeroScore": 0, "shenqi_pos": ({
    0,
    0,
}), "cOrg": "", "iPride": 733, "iHp_Max": 464, "iSpe_All": 79, "fabao": ([]), "iDamage_All": 107, "iMarry": 0, "outdoor_level": 0, "iCash": 32029, "shenqi": ([]), "iSkiPoint": 0, "iDex_All": 79, "iBeastSki2": 0, "usernum": 20759973, "iIcon": 9, "addPoint": 0, "commu_name": 0, "iDesc": 0, "iSchool": 7, "iGoodness": 0, "igoodness_sav": 0, "iStr_All": 80, "AchPointTotal": 91, "iCor_All": 81, "jiyuan": 0, "iCGBodyAmount": 0, "iAtt_All": 166, "iZhuanZhi": 0, "total_horse": 0, "iMagDef_All": 155, "iCGTotalAmount": 0, "iMp_Max": 339, "iSmithski": 0, "iPcktPage": 0, "iRace": 3, "TA_iAllPoint": 0, "iExptSki1": 0, "ori_desc": 146, "ExpJw": 0, "ExAvt": ([]), "i3FlyLv": 0, "rent_level": 0, "ori_race": 3, "iMaxExpt1": 20, "TA_iAllNewPoint": 0, "iBeastSki3": 0, "changesch": ({}), "AllRider": ([]), "iErrantry": 0, "iHp": 454, "iGrade": 69, "pet": ({}), "iMaxExpt3": 20, "iSaving": 0, "shenqi_yellow": "", "xianyu": 0, "iHp_Eff": 464, "energy": 0, "iMaxExpt4": 20, "iExptSki3": 0, "iUpExp": 1084477, "propKept": ([]), "farm_level": 0, "iExptSki5": 0, ])`;

const obj = JSON.parse(failedJson
    .split(/\r?\n?/).join('') // make it a single line so things work
    .split('([').join('{') // convert ([ to {
    .split('])').join('}') // convert ]) to }
    .split('({').join('[') // convert ({ to [
    .split('})').join(']') // convert }) to ]
    .split(/,\s*}/g).join('}') // convert ,} to } because trailing , are invalid in JSON
    .split(/,\s*\]/g).join(']') // convert ,] to ] because trailing , are invalid in JSON
);
console.log(JSON.stringify(obj, null, 4));

You could do the above with .replace as well, but because [ and ( are "special" characters that need to be escaped in a regex, the code is far less clear in my opinion.

Upvotes: 1

petezurich
petezurich

Reputation: 10194

Converting Jaromanda X's code to Python results in this:

data = " ... your string here ... "

data = data.replace("\r", "")
data = data.replace("\n", "")
data = data.replace("\t", "")
data = data.replace("    ","")
data = data.replace("([", "{")
data = data.replace("])", "}")
data = data.replace("({", "[")
data = data.replace("})", "]")
data = data.replace(", }", "}")  
data = data.replace(", ]", "]")  

final_json = json.loads(data)

Printout:

{'icolor_ex': 0,
 'iDod_All': 91,
 'commu_gid': 0,
 'normal_horse': 0,
 'iBeastSki1': 0,
 'rent': 0,
 'datang_feat': 452,
 'total_avatar': 0,
 'iDef_All': 188,  
 ...

Upvotes: 0

Related Questions