1. JSON数据处理基础
JSON作为现代应用开发中最常用的数据交换格式，其轻量级和易读性使其成为API交互的首选。Python内置的json模块提供了完整的JSON处理能力，我们先从最基础的解析开始。
1.1 JSON字符串解析
json.loads()是将JSON字符串转换为Python对象的核心方法。实际开发中需要注意几个关键点：
python复制import json
# 标准JSON字符串解析
json_str = '{"name": "Alice", "age": 25, "is_active": true}'
data = json.loads(json_str)
# 类型转换对照
print(type(data["name"])) # <class 'str'>
print(type(data["age"])) # <class 'int'>
print(type(data["is_active"])) # <class 'bool'>
# 处理嵌套结构
complex_json = '''
{
"users": [
{"id": 1, "name": "Alice"},
{"id": 2, "name": "Bob"}
],
"meta": {
"page": 1,
"total": 100
}
}
'''
parsed_data = json.loads(complex_json)
print(parsed_data["users"][0]["name"]) # Alice
注意：JSON中的true/false/null会分别转换为Python的True/False/None，数字会根据值自动转为int或float
1.2 文件读写操作
实际项目中更多是处理JSON文件而非字符串。文件操作时需特别注意编码问题：
python复制# 写入JSON文件的最佳实践
data = {
"server": {
"host": "127.0.0.1",
"ports": [8000, 8001, 8002]
},
"timeout": 30.5
}
with open("config.json", "w", encoding="utf-8") as f:
json.dump(data, f,
indent=2, # 缩进美化
ensure_ascii=False, # 允许非ASCII字符
sort_keys=True) # 键排序
# 读取JSON文件的安全写法
try:
with open("config.json", "r", encoding="utf-8") as f:
config = json.load(f)
print(config["server"]["ports"][0]) # 8000
except FileNotFoundError:
print("配置文件不存在")
except json.JSONDecodeError:
print("配置文件格式错误")
1.3 特殊数据类型处理
JSON标准数据类型有限，处理Python特有类型时需要自定义编码器：
python复制from datetime import datetime
from decimal import Decimal
class EnhancedJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime):
return obj.isoformat()
if isinstance(obj, Decimal):
return float(obj)
if isinstance(obj, set):
return list(obj)
return super().default(obj)
# 使用自定义编码器
data = {
"timestamp": datetime.now(),
"price": Decimal("19.99"),
"tags": {"python", "json", "xml"}
}
json_str = json.dumps(data, cls=EnhancedJSONEncoder)
print(json_str)
# 输出示例：{"timestamp": "2023-07-20T14:30:00.123456", "price": 19.99, "tags": ["xml", "python", "json"]}
2. XML数据处理实战
XML在配置文件和企业级数据交换中仍占据重要地位。Python标准库中的xml.etree.ElementTree提供了轻量级XML处理方案。
2.1 基本解析方法
python复制import xml.etree.ElementTree as ET
# 解析XML字符串
xml_data = '''
<inventory>
<item id="1001">
<name>笔记本电脑</name>
<spec>
<cpu>i7-12700H</cpu>
<memory>16GB</memory>
</spec>
<price currency="CNY">6999.00</price>
</item>
</inventory>
'''
root = ET.fromstring(xml_data)
# 获取元素属性
item = root.find("item")
print(item.get("id")) # 1001
# 获取元素文本
print(item.find("name").text) # 笔记本电脑
# 处理嵌套元素
spec = item.find("spec")
print(spec.find("cpu").text) # i7-12700H
2.2 高级查询技巧
XPath表达式可以大幅简化复杂XML的查询：
python复制# 查找所有价格大于5000的商品
for item in root.findall(".//item[price>5000]"):
name = item.find("name").text
price = item.find("price").text
print(f"高端商品: {name}, 价格: {price}")
# 获取特定属性的元素
currency = root.find(".//price[@currency='CNY']")
print(currency.text) # 6999.00
2.3 XML文件生成
创建XML文档时建议使用ElementTree的API而非字符串拼接：
python复制# 创建完整的XML文档
root = ET.Element("catalog")
root.set("version", "1.0")
# 添加子元素
book1 = ET.SubElement(root, "book")
book1.set("id", "b001")
ET.SubElement(book1, "title").text = "Python高级编程"
ET.SubElement(book1, "author").text = "李四"
ET.SubElement(book1, "year").text = "2023"
# 添加带属性的元素
price = ET.SubElement(book1, "price")
price.text = "89.00"
price.set("currency", "CNY")
# 生成XML文件
tree = ET.ElementTree(root)
tree.write("books.xml",
encoding="utf-8",
xml_declaration=True,
short_empty_elements=False)
3. 性能优化与最佳实践
3.1 大文件处理技巧
处理大型JSON/XML文件时需要特别注意内存使用：
python复制# 流式处理大型JSON文件
import ijson
with open("large_data.json", "rb") as f:
for item in ijson.items(f, "items.item"):
process_item(item) # 逐项处理
# 增量解析大型XML文件
for event, elem in ET.iterparse("large_data.xml", events=("end",)):
if elem.tag == "item":
process_item(elem)
elem.clear() # 及时释放内存
3.2 数据验证与清洗
python复制# JSON Schema验证
from jsonschema import validate
schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "number", "minimum": 0}
},
"required": ["name"]
}
data = {"name": "Alice", "age": 25}
validate(instance=data, schema=schema)
# XML Schema验证
from lxml import etree
xmlschema = etree.XMLSchema(file="schema.xsd")
xml_doc = etree.parse("data.xml")
xmlschema.assertValid(xml_doc)
3.3 格式转换技巧
python复制# JSON转XML
def json_to_xml(json_obj, line_padding=""):
result = []
if isinstance(json_obj, dict):
for key, value in json_obj.items():
result.append(f"{line_padding}<{key}>")
result.append(json_to_xml(value, line_padding + "\t"))
result.append(f"{line_padding}</{key}>")
elif isinstance(json_obj, list):
for item in json_obj:
result.append(json_to_xml(item, line_padding))
else:
return f"{line_padding}{json_obj}"
return "\n".join(result)
# XML转JSON
def xml_to_json(xml_str):
root = ET.fromstring(xml_str)
return {root.tag: parse_element(root)}
def parse_element(element):
result = {}
for child in element:
if len(child) > 0:
result[child.tag] = parse_element(child)
else:
result[child.tag] = child.text
return result
4. 常见问题排查
4.1 JSON处理典型错误
python复制# 编码问题
try:
data = json.loads('{"name": "中文"}')
except json.JSONDecodeError as e:
print(f"解析错误: {e.msg} (位置: {e.pos})")
# 类型错误
try:
json.dumps({"date": datetime.now()})
except TypeError as e:
print(f"序列化错误: {str(e)}")
# 解决方案: 使用自定义编码器
# 循环引用
data = {}
data["self"] = data
try:
json.dumps(data)
except ValueError as e:
print(f"循环引用错误: {str(e)}")
4.2 XML处理陷阱
python复制# 命名空间问题
ns_xml = '''
<root xmlns:ns="http://example.com">
<ns:item>value</ns:item>
</root>
'''
root = ET.fromstring(ns_xml)
# 错误方式
# print(root.find("item").text) # 返回None
# 正确方式
namespaces = {"ns": "http://example.com"}
print(root.find("ns:item", namespaces).text) # value
# 实体引用问题
malicious_xml = '''
<!DOCTYPE test [
<!ENTITY xxe SYSTEM "file:///etc/passwd">
]>
<test>&xxe;</test>
'''
# 危险操作
# root = ET.fromstring(malicious_xml) # 可能泄露敏感信息
# 安全方式
parser = ET.XMLParser(resolve_entities=False)
root = ET.fromstring(malicious_xml, parser=parser)
4.3 性能优化检查点
python复制# JSON性能对比
import timeit
setup = '''
import json
data = [{"id": i, "value": "test"*100} for i in range(1000)]
'''
print("json.dumps:", timeit.timeit('json.dumps(data)', setup, number=100))
print("ujson.dumps:", timeit.timeit('import ujson; ujson.dumps(data)', setup, number=100))
# XML解析器选择
xml_data = "<root>" + "<item>test</item>"*1000 + "</root>"
print("ElementTree:", timeit.timeit('ET.fromstring(xml_data)',
'import xml.etree.ElementTree as ET;' + f'xml_data="{xml_data}"',
number=100))
print("lxml:", timeit.timeit('etree.fromstring(xml_data)',
'from lxml import etree;' + f'xml_data="{xml_data}"',
number=100))
5. 实际应用案例
5.1 API数据处理
python复制# 处理REST API响应
import requests
def get_weather(city):
url = f"https://api.weather.com/v1/{city}"
try:
response = requests.get(url, timeout=5)
response.raise_for_status()
data = response.json()
# 数据提取与转换
return {
"city": data["location"]["name"],
"temp": data["current"]["temp_c"],
"condition": data["current"]["condition"]["text"]
}
except requests.exceptions.RequestException as e:
print(f"API请求失败: {str(e)}")
return None
except (KeyError, json.JSONDecodeError) as e:
print(f"数据解析错误: {str(e)}")
return None
5.2 配置文件管理
python复制# 智能配置加载器
class ConfigLoader:
def __init__(self):
self.config = {}
def load(self, file_path):
if file_path.endswith(".json"):
with open(file_path, "r") as f:
self.config.update(json.load(f))
elif file_path.endswith(".xml"):
tree = ET.parse(file_path)
root = tree.getroot()
self.config.update(self._parse_xml(root))
else:
raise ValueError("不支持的配置文件格式")
def _parse_xml(self, element):
result = {}
for child in element:
if len(child) > 0:
result[child.tag] = self._parse_xml(child)
else:
result[child.tag] = child.text
return result
def get(self, key, default=None):
keys = key.split(".")
value = self.config
try:
for k in keys:
value = value[k]
return value
except (KeyError, TypeError):
return default
5.3 数据转换管道
python复制# 数据格式转换中间件
class DataTransformer:
@staticmethod
def transform(input_data, input_format, output_format):
# 统一转换为中间字典格式
if input_format == "json":
data = json.loads(input_data)
elif input_format == "xml":
root = ET.fromstring(input_data)
data = DataTransformer._xml_to_dict(root)
else:
raise ValueError("不支持的输入格式")
# 转换为目标格式
if output_format == "json":
return json.dumps(data, indent=2)
elif output_format == "xml":
root = DataTransformer._dict_to_xml(data)
return ET.tostring(root, encoding="unicode")
else:
raise ValueError("不支持的输出格式")
@staticmethod
def _xml_to_dict(element):
result = {}
for child in element:
if len(child) > 0:
result[child.tag] = DataTransformer._xml_to_dict(child)
else:
result[child.tag] = child.text
return result
@staticmethod
def _dict_to_xml(data, parent=None):
if parent is None:
parent = ET.Element("root")
if isinstance(data, dict):
for key, value in data.items():
element = ET.SubElement(parent, key)
DataTransformer._dict_to_xml(value, element)
elif isinstance(data, list):
for item in data:
DataTransformer._dict_to_xml(item, parent)
else:
parent.text = str(data)
return parent