【录制回放】基于HAR包的流量录制回放¶
基于HAR包的流量录制回放¶
什么是HAR包？¶
HAR（HTTP Archive format），是一种JSON格式的存档格式文件，通用扩展名为
.har。
HAR包是JSON格式的，打开后，重点关注entries里面的request和response，包含了请求和响应信息。
流量录制¶
怎么获取HAR包呢？可以网上搜索方法，浏览器F12、抓包工具（Charles、Proxyman等）都可以将HTTP请求导出为HAR包。
生产级别的流量录制很复杂，想要了解的话可以参考阿里开源项目：https://github.com/alibaba/jvm-sandbox-repeater
回放对比¶
本文重点介绍在导出HAR包后，怎么通过Python来实现回放对比。
使用介绍¶
一、将HAR包转换为pytest用例
har_file为har包路径，profile配置开启回放，调用Har.har2case()方法将HAR包转换为pytest用例。
转换后会生成：
demo_test.py 与HAR同名的pytest用例文件
demo-replay-diff 对比结果目录，暂时为空
sqlite.db 存储HAR包响应数据，标记为expect
二、执行pytest用例
from tep.libraries.Diff import Diff
from tep.libraries.Sqlite import Sqlite
def test(HTTPRequestKeyword, BodyKeyword):
var = {'caseId': '023f387db398504889e0afb8dd1bc99d', 'requestOrder': 1, 'diffDir': '/Users/wanggang424/Desktop/PycharmProjects/tep/tests/demo/case/har/demo-replay-diff'}
url = "https://postman-echo.com/get?foo1=HDnY8&foo2=34.5"
headers = {'Host': 'postman-echo.com', 'User-Agent': 'HttpRunnerPlus', 'Accept-Encoding': 'gzip'}
ro = HTTPRequestKeyword("get", url=url, headers=headers)
# user_defined_var = ro.response.jsonpath("$.jsonpath")
assert ro.response.status_code < 400
Sqlite.record_actual((ro.response.text, var["caseId"], var["requestOrder"], "GET", "https://postman-echo.com/get?foo1=HDnY8&foo2=34.5"), var)
url = "https://postman-echo.com/post"
headers = {'Host': 'postman-echo.com', 'User-Agent': 'Go-http-client/1.1', 'Content-Length': '28', 'Content-Type': 'application/json; charset=UTF-8', 'Cookie': 'sails.sid=s%3Az_LpglkKxTvJ_eHVUH6V67drKp0AGWW-.PidabaXOnatLRP47hVyqqepl6BdrpEQzRlJQXtbIiwk', 'Accept-Encoding': 'gzip', 'sails.sid': 's%3Az_LpglkKxTvJ_eHVUH6V67drKp0AGWW-.PidabaXOnatLRP47hVyqqepl6BdrpEQzRlJQXtbIiwk'}
body = r"""{"foo1":"HDnY8","foo2":12.3}"""
ro = BodyKeyword(body)
# ro = BodyKeyword(body, {"$.jsonpath": "value"})
body = ro.data
ro = HTTPRequestKeyword("post", url=url, headers=headers, json=body)
# user_defined_var = ro.response.jsonpath("$.jsonpath")
assert ro.response.status_code < 400
Sqlite.record_actual((ro.response.text, var["caseId"], var["requestOrder"], "POST", "https://postman-echo.com/post"), var)
url = "https://postman-echo.com/post"
headers = {'Host': 'postman-echo.com', 'User-Agent': 'Go-http-client/1.1', 'Content-Length': '20', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie': 'sails.sid=s%3AS5e7w0zQ0xAsCwh9L8T6R7QLYCO7_gtD.r8%2B2w9IWqEIfuVkrZjnxzm2xADIk34zKAWXRPapr%2FAw', 'Accept-Encoding': 'gzip', 'sails.sid': 's%3AS5e7w0zQ0xAsCwh9L8T6R7QLYCO7_gtD.r8%2B2w9IWqEIfuVkrZjnxzm2xADIk34zKAWXRPapr%2FAw'}
ro = HTTPRequestKeyword("post", url=url, headers=headers, data=body)
# user_defined_var = ro.response.jsonpath("$.jsonpath")
assert ro.response.status_code < 400
Sqlite.record_actual((ro.response.text, var["caseId"], var["requestOrder"], "POST", "https://postman-echo.com/post"), var)
Diff.make(var["caseId"], var["diffDir"])
执行时会记录实际请求响应，存入sqlite数据库，标记为actual。
同时进行JSON字段对比和文本对比，分别生成字段对比.txt和文本对比.html。
原理解析¶
源码：https://github.com/dongfanger/tep.git
一、转换
通过haralyzer库解析HAR包，获取到request和response，再拼装成pytest用例。
实现文件：tep/libraries/Har.py
def _make_case(self):
var = self._prepare_var()
steps = self._prepare_steps()
content = Har.TEMPLATE.format(var=var, steps=steps)
if self.replay:
content = Har.TEMPLATE_IMPORT_REPLAY + content
if not os.path.exists(self.replay_dff_dir):
os.makedirs(self.replay_dff_dir)
content += Har.TEMPLATE_DIFF
with open(self.case_file, "w") as f:
f.write(content)
def _prepare_step(self, entry) -> list:
logger.info("{} {} Convert step", entry.request.method, entry.request.url)
step = Step()
self._make_request_method(step, entry)
self._make_request_url(step, entry)
self._make_request_headers(step, entry)
self._make_request_body(step, entry)
self._make_before_param(step, entry)
self._make_after_extract(step, entry)
self._make_after_assert(step, entry)
if self.replay:
self._make_after_replay(step, entry)
self._save_replay(step, entry) # Save replay data to sqlite
return self._make_statement(step, entry)
def _request_param(self, step, entry) -> str:
param = ""
mime_type = entry.request.mimeType
b = 'data=body' if mime_type and mime_type.startswith("application/x-www-form-urlencoded") else 'json=body'
if step.request.method == "GET":
param = '"get", url=url, headers=headers, params=body' if step.request.body else '"get", url=url, headers=headers'
if step.request.method == "POST":
param = '"post", url=url, headers=headers, '
param += b
if step.request.method == "PUT":
param = '"put", url=url, headers=headers, '
param += b
if step.request.method == "DELETE":
param = '"delete", url=url, headers=headers'
if self.profile.get("http2", False):
param += ', http2=True'
return param
转换过程中，将响应text存入sqlite数据库中：
def _save_replay(self, step, entry):
Sqlite.create_table_replay()
data = (
self.case_id,
self.request_order,
step.request.method,
step.request.url,
self._decode_text(entry)
)
Sqlite.insert_into_replay_expect(data)
self.request_order += 1
二、存储
通过sqlite数据库存储。
实现文件：tep/libraries/Sqlite.py
class Sqlite:
DB_FILE = "sqlite3.db"
@staticmethod
def execute(sql: str, data: tuple = None):
os.chdir(Config.BASE_DIR)
conn = sqlite3.connect(Sqlite.DB_FILE)
if data:
conn.execute(sql, data)
else:
conn.execute(sql)
conn.commit()
conn.close()
@staticmethod
def create_table_replay():
Sqlite.execute("""CREATE TABLE IF NOT EXISTS replay
(id INTEGER PRIMARY KEY AUTOINCREMENT,
case_id TEXT NOT NULL,
request_order INTEGER NOT NULL,
method TEXT NOT NULL,
url TEXT NOT NULL,
expect TEXT NOT NULL,
actual TEXT)""")
@staticmethod
def insert_into_replay_expect(data: tuple):
if not Sqlite.is_replay_existed(data):
Sqlite.execute("INSERT INTO replay(case_id, request_order, method, url, expect) VALUES (?, ?, ?, ?, ?)", data)
三、记录
通过profile开关控制是否开启回放，对比开启前后用例差异：
开启回放，是在每个步骤后置中，添加了Sqlite记录响应text，并在最后执行Diff。
记录实现文件：tep/libraries/Sqlite.py
@staticmethod
def update_replay_actual(data: tuple):
Sqlite.execute("UPDATE replay SET actual = ? WHERE case_id = ? AND request_order = ? AND method = ? AND url = ?", data)
@staticmethod
def record_actual(data: tuple, var: dict):
Sqlite.update_replay_actual(data)
var["requestOrder"] += 1
四、对比
做了2个对比，一是通过deepdiff库进行JSON字段对比，二是通过difflib库进行文本对比。
实现文件：tep/libraries/Diff.py
1、JSON字段对比，每个请求对比结果放入列表中，输出到TXT文本
2、文本对比，从数据库取出expect和actual并格式化，所有响应text拼接到一个字符串进行对比，输出到HTML文件
@staticmethod
def make(case_id: str, diff_dir: str):
results = Sqlite.get_expect_actual(case_id)
diffs = []
expect_text = ""
actual_text = ""
for row in results:
expect, actual, method, url = row
diff = Diff.make_deepdiff(expect, actual)
diffs.append(diff)
expect_text += Diff._format_json_str(expect, method, url)
actual_text += Diff._format_json_str(actual, method, url)
diff_html = Diff.make_difflib(expect_text, actual_text)
Diff._output_file(diffs, diff_html, diff_dir)
@staticmethod
def make_deepdiff(expect: str, actual: str) -> str:
diff = DeepDiff(expect, actual)
return str(diff)
@staticmethod
def make_difflib(lines1: str, lines2: str) -> str:
diff = difflib.HtmlDiff().make_file(lines1.splitlines(), lines2.splitlines())
return diff
@staticmethod
def _format_json_str(text: str, method: str, url: str) -> str:
s = json.dumps(json.loads(text), indent=4, ensure_ascii=False)
s = "{} {}\n{}\n\n".format(method, url, s)
return s
@staticmethod
def _output_file(diffs: list, diff_html: str, diff_dir: str):
with open(os.path.join(diff_dir, Diff.DIFF_TXT_FILE), "w") as f:
f.write("\n\n".join(diffs))
styled_html = f"<style>td {{width: 50%;}}</style>\n{diff_html}"
with open(os.path.join(diff_dir, Diff.DIFF_HTML_FILE), "w") as f:
f.write(styled_html)
流量录制回放已经成为一种重要的测试手段，既可以快速生成自动化用例，也可以回放对比开发和线上差异，学习起来吧。