735 lines
30 KiB
Python
735 lines
30 KiB
Python
|
"""部署脚本（支持 .env/.json 配置）
|
|||
|
|
实现以下规范：
|
|||
|
|
- 构建：Maven/Gradle clean 并跳过测试，生成可执行 jar
|
|||
|
|
- 严格校验：校验 BOOT-INF/classes 配置文件与 MANIFEST 元数据
|
|||
|
|
- 安全传输：SCP 非 22 端口 + 指纹校验 + MD5 验证
|
|||
|
|
- 预检查：JDK17+、磁盘空间 >= 2GB、ulimit 满足 SpringBoot3
|
|||
|
|
- 进程管理：优雅停止（SIGTERM 30s）超时强杀（SIGKILL）
|
|||
|
|
- 备份保留：/opt/app/backups 保留最近 5 个版本
|
|||
|
|
- 启动与健康：nohup 启动、120s 日志无 ERROR、actuator/health=200
|
|||
|
|
- 回滚与告警：自动回滚最近可用版本并发送告警
|
|||
|
|
- 日志：JSON Lines 记录所有步骤与环境信息
|
|||
|
|
|
|||
|
|
配置优先级：
|
|||
|
|
- 优先读取 DEPLOY_CONFIG 指定的路径（.env 或 .json）
|
|||
|
|
- 未指定时优先加载同目录下 deployment.env，其次 deployment.config.json
|
|||
|
|
"""
|
|||
|
|
import base64
|
|||
|
|
import datetime
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
import shutil
|
|||
|
|
import socket
|
|||
|
|
import subprocess
|
|||
|
|
import sys
|
|||
|
|
import time
|
|||
|
|
import urllib.request
|
|||
|
|
import urllib.error
|
|||
|
|
import zipfile
|
|||
|
|
import paramiko
|
|||
|
|
from types import SimpleNamespace
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _ts_ms():
|
|||
|
|
"""返回当前时间戳（毫秒）"""
|
|||
|
|
return int(time.time() * 1000)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _log_path():
|
|||
|
|
"""返回部署日志文件路径（JSONL 格式）"""
|
|||
|
|
return os.path.join(os.path.dirname(__file__), "deployment.log.jsonl")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def log_step(step, status, details=None, extra=None, env=None):
|
|||
|
|
"""记录部署步骤到 JSONL 日志
|
|||
|
|
step：步骤名称；status：ok/fail/skip 等；
|
|||
|
|
details/extra：附加信息；env：环境信息（服务器、目录等）
|
|||
|
|
"""
|
|||
|
|
rec = {
|
|||
|
|
"timestamp_ms": _ts_ms(),
|
|||
|
|
"step": step,
|
|||
|
|
"status": status,
|
|||
|
|
"details": details or "",
|
|||
|
|
"extra": extra or {},
|
|||
|
|
}
|
|||
|
|
if env:
|
|||
|
|
rec["env"] = env
|
|||
|
|
with open(_log_path(), "a", encoding="utf-8") as f:
|
|||
|
|
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def print_progress(percent, step, status="进行中", details=None):
|
|||
|
|
try:
|
|||
|
|
msg = f"[{str(percent).rjust(3)}%] {step} - {status}"
|
|||
|
|
if details:
|
|||
|
|
msg += f" | {details}"
|
|||
|
|
print(msg, flush=True)
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
|
|||
|
|
def run_cmd(cmd, cwd=None, env=None, capture_output=True, timeout=None):
|
|||
|
|
"""运行本地命令并返回 (返回码, 标准输出, 标准错误)"""
|
|||
|
|
try:
|
|||
|
|
p = subprocess.run(
|
|||
|
|
cmd,
|
|||
|
|
shell=True,
|
|||
|
|
cwd=cwd,
|
|||
|
|
env=env,
|
|||
|
|
stdout=subprocess.PIPE if capture_output else None,
|
|||
|
|
stderr=subprocess.PIPE if capture_output else None,
|
|||
|
|
timeout=timeout,
|
|||
|
|
)
|
|||
|
|
out = p.stdout.decode("utf-8", errors="replace") if capture_output and p.stdout else ""
|
|||
|
|
err = p.stderr.decode("utf-8", errors="replace") if capture_output and p.stderr else ""
|
|||
|
|
return p.returncode, out, err
|
|||
|
|
except subprocess.TimeoutExpired as te:
|
|||
|
|
out = te.stdout.decode("utf-8", errors="replace") if capture_output and te.stdout else ""
|
|||
|
|
err = te.stderr.decode("utf-8", errors="replace") if capture_output and te.stderr else "command timeout"
|
|||
|
|
return 124, out, err
|
|||
|
|
|
|||
|
|
|
|||
|
|
def md5_file(path):
|
|||
|
|
"""计算本地文件 MD5 摘要"""
|
|||
|
|
h = hashlib.md5()
|
|||
|
|
with open(path, "rb") as f:
|
|||
|
|
for chunk in iter(lambda: f.read(8192), b""):
|
|||
|
|
h.update(chunk)
|
|||
|
|
return h.hexdigest()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sha256_bytes(b):
|
|||
|
|
"""计算字节的 SHA256 并返回 base64 字符串（用于指纹）"""
|
|||
|
|
return base64.b64encode(hashlib.sha256(b).digest()).decode("ascii")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def find_jar(project_dir, build_tool, target_jar_name=None):
|
|||
|
|
"""定位构建产物 jar 文件（支持指定文件名或自动选择最新）"""
|
|||
|
|
if target_jar_name:
|
|||
|
|
maven_path = os.path.join(project_dir, "target", target_jar_name)
|
|||
|
|
gradle_path = os.path.join(project_dir, "build", "libs", target_jar_name)
|
|||
|
|
if os.path.exists(maven_path):
|
|||
|
|
return maven_path
|
|||
|
|
if os.path.exists(gradle_path):
|
|||
|
|
return gradle_path
|
|||
|
|
return target_jar_name
|
|||
|
|
if build_tool == "maven":
|
|||
|
|
target = os.path.join(project_dir, "target")
|
|||
|
|
else:
|
|||
|
|
target = os.path.join(project_dir, "build", "libs")
|
|||
|
|
if not os.path.isdir(target):
|
|||
|
|
return None
|
|||
|
|
jars = []
|
|||
|
|
for name in os.listdir(target):
|
|||
|
|
if name.endswith(".jar"):
|
|||
|
|
jars.append(os.path.join(target, name))
|
|||
|
|
if not jars:
|
|||
|
|
return None
|
|||
|
|
jars.sort(key=lambda p: os.path.getmtime(p), reverse=True)
|
|||
|
|
return jars[0]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def verify_jar_integrity(jar_path):
|
|||
|
|
"""校验 jar 完整性（支持 Boot Jar 与 Boot War）：
|
|||
|
|
- 必须存在 application.yml、application-prod.yml、logback-plus.xml（在 classes 下）
|
|||
|
|
- MANIFEST 必须包含 Start-Class / Implementation-Version / Build-Time
|
|||
|
|
"""
|
|||
|
|
z = zipfile.ZipFile(jar_path)
|
|||
|
|
names = set(z.namelist())
|
|||
|
|
prefixes = ["BOOT-INF/classes/", "WEB-INF/classes/"]
|
|||
|
|
files = ["application.yml", "application-prod.yml", "logback-plus.xml"]
|
|||
|
|
missing = []
|
|||
|
|
for f in files:
|
|||
|
|
if not any((p + f) in names for p in prefixes):
|
|||
|
|
missing.append(f)
|
|||
|
|
ok = len(missing) == 0
|
|||
|
|
manifest = None
|
|||
|
|
if "META-INF/MANIFEST.MF" in names:
|
|||
|
|
manifest = z.read("META-INF/MANIFEST.MF").decode("utf-8", errors="replace")
|
|||
|
|
else:
|
|||
|
|
ok = False
|
|||
|
|
if manifest:
|
|||
|
|
has_start_class = bool(re.search(r"^Start-Class:\s*.+", manifest, re.MULTILINE))
|
|||
|
|
has_impl_ver = bool(re.search(r"^Implementation-Version:\s*.+", manifest, re.MULTILINE))
|
|||
|
|
has_build_time = bool(re.search(r"^Build-Time:\s*.+", manifest, re.MULTILINE))
|
|||
|
|
if not (has_start_class and has_impl_ver and has_build_time):
|
|||
|
|
ok = False
|
|||
|
|
if not has_start_class:
|
|||
|
|
missing.append("Start-Class")
|
|||
|
|
if not has_impl_ver:
|
|||
|
|
missing.append("Implementation-Version")
|
|||
|
|
if not has_build_time:
|
|||
|
|
missing.append("Build-Time")
|
|||
|
|
z.close()
|
|||
|
|
return ok, missing
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_manifest_info(jar_path):
|
|||
|
|
"""解析 MANIFEST.MF 中的关键字段信息"""
|
|||
|
|
z = zipfile.ZipFile(jar_path)
|
|||
|
|
info = {}
|
|||
|
|
if "META-INF/MANIFEST.MF" in z.namelist():
|
|||
|
|
m = z.read("META-INF/MANIFEST.MF").decode("utf-8", errors="replace")
|
|||
|
|
for k in ["Start-Class", "Implementation-Version", "Build-Time"]:
|
|||
|
|
mobj = re.search(rf"^{k}:\s*(.+)$", m, re.MULTILINE)
|
|||
|
|
if mobj:
|
|||
|
|
info[k] = mobj.group(1).strip()
|
|||
|
|
z.close()
|
|||
|
|
return info
|
|||
|
|
|
|||
|
|
|
|||
|
|
def maven_build(project_dir, maven_home):
|
|||
|
|
"""执行 Maven 构建（跳过测试）"""
|
|||
|
|
mvn = os.path.join(maven_home, "bin", "mvn.cmd") if os.name == "nt" else os.path.join(maven_home, "bin", "mvn")
|
|||
|
|
cmd = f'"{mvn}" clean package -DskipTests=true'
|
|||
|
|
return run_cmd(cmd, cwd=project_dir, capture_output=False)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def gradle_build(project_dir, gradle_home=None):
|
|||
|
|
"""执行 Gradle 构建（跳过测试，优先使用 gradlew）"""
|
|||
|
|
gradlew = os.path.join(project_dir, "gradlew.cmd") if os.name == "nt" else os.path.join(project_dir, "gradlew")
|
|||
|
|
if os.path.exists(gradlew):
|
|||
|
|
cmd = f'"{gradlew}" clean build -x test'
|
|||
|
|
return run_cmd(cmd, cwd=project_dir, capture_output=False)
|
|||
|
|
if gradle_home:
|
|||
|
|
gradle_bin = os.path.join(gradle_home, "bin", "gradle")
|
|||
|
|
if os.name == "nt":
|
|||
|
|
gradle_bin += ".bat"
|
|||
|
|
cmd = f'"{gradle_bin}" clean build -x test'
|
|||
|
|
return run_cmd(cmd, cwd=project_dir, capture_output=False)
|
|||
|
|
return 1, "", "gradle not found"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def connect_ssh(host, port, user, password=None, keyfile=None):
|
|||
|
|
"""建立 SSH 连接（密码或密钥两种方式）"""
|
|||
|
|
ssh = paramiko.SSHClient()
|
|||
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|||
|
|
if keyfile:
|
|||
|
|
ssh.connect(host, port=port, username=user, key_filename=keyfile)
|
|||
|
|
else:
|
|||
|
|
ssh.connect(host, port=port, username=user, password=password)
|
|||
|
|
try:
|
|||
|
|
ssh.get_transport().set_keepalive(15)
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
return ssh
|
|||
|
|
|
|||
|
|
|
|||
|
|
def ensure_fingerprint(host, port, user, password, keyfile, expect_fp):
|
|||
|
|
"""校验服务器指纹（要求与配置的 SHA256 指纹一致）"""
|
|||
|
|
ssh = connect_ssh(host, port, user, password, keyfile)
|
|||
|
|
k = ssh.get_transport().get_remote_server_key()
|
|||
|
|
fp = "SHA256:" + sha256_bytes(base64.b64decode(k.get_base64()))
|
|||
|
|
ssh.close()
|
|||
|
|
return fp == expect_fp, fp
|
|||
|
|
|
|||
|
|
|
|||
|
|
def scp_transfer(local_path, host, port, user, remote_path, timeout=None):
|
|||
|
|
"""使用 scp 非标准端口传输文件到远端目录"""
|
|||
|
|
cmd = (
|
|||
|
|
f'scp -o BatchMode=yes '
|
|||
|
|
f'-o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=3 '
|
|||
|
|
f'-o StrictHostKeyChecking=accept-new -P {port} "{local_path}" {user}@{host}:"{remote_path}"'
|
|||
|
|
)
|
|||
|
|
return run_cmd(cmd, timeout=timeout)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sftp_transfer(ssh, local_path, remote_path):
|
|||
|
|
sftp = ssh.open_sftp()
|
|||
|
|
try:
|
|||
|
|
sftp.put(local_path, remote_path)
|
|||
|
|
return 0, "", ""
|
|||
|
|
finally:
|
|||
|
|
try:
|
|||
|
|
sftp.close()
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
|
|||
|
|
def ssh_exec(ssh, cmd, timeout=None, print_output=True):
|
|||
|
|
"""执行远端命令并返回 (返回码, 标准输出, 标准错误)"""
|
|||
|
|
if print_output:
|
|||
|
|
print(f'SSH$ {cmd}', flush=True)
|
|||
|
|
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
|
|||
|
|
out = stdout.read().decode("utf-8", errors="replace")
|
|||
|
|
err = stderr.read().decode("utf-8", errors="replace")
|
|||
|
|
rc = stdout.channel.recv_exit_status()
|
|||
|
|
if print_output:
|
|||
|
|
if out:
|
|||
|
|
try:
|
|||
|
|
print(out if out.endswith("\n") else out + "\n", end="", flush=True)
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
if err:
|
|||
|
|
try:
|
|||
|
|
print(err if err.endswith("\n") else err + "\n", end="", flush=True)
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
return rc, out, err
|
|||
|
|
|
|||
|
|
|
|||
|
|
def remote_md5(ssh, path):
|
|||
|
|
"""获取远端文件摘要（优先 md5sum，失败则尝试 sha256sum）"""
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f'md5sum "{path}" || sha256sum "{path}"')
|
|||
|
|
algo = "md5"
|
|||
|
|
if " " in out.strip():
|
|||
|
|
val = out.strip().split()[0]
|
|||
|
|
if rc != 0:
|
|||
|
|
return None, algo
|
|||
|
|
return val, algo
|
|||
|
|
return None, algo
|
|||
|
|
|
|||
|
|
|
|||
|
|
def pre_checks(ssh, remote_dir):
|
|||
|
|
ok = True
|
|||
|
|
details = {}
|
|||
|
|
rc, out, _ = ssh_exec(ssh, "java -version")
|
|||
|
|
version_match = re.search(r'version \"(?P<v>[\\d\\.]+)', out) or re.search(r'openjdk version \"(?P<v>[\\d\\.]+)', out)
|
|||
|
|
if version_match:
|
|||
|
|
major = int(version_match.group("v").split(".")[0])
|
|||
|
|
details["java_major"] = major
|
|||
|
|
details["java_ok"] = major >= 17
|
|||
|
|
ok = ok and details["java_ok"]
|
|||
|
|
else:
|
|||
|
|
details["java_ok"] = False
|
|||
|
|
ok = False
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f'df -Pk "{remote_dir}" | tail -1')
|
|||
|
|
parts = out.split()
|
|||
|
|
if len(parts) >= 4:
|
|||
|
|
try:
|
|||
|
|
avail_k = int(parts[3])
|
|||
|
|
except:
|
|||
|
|
avail_k = 0
|
|||
|
|
details["disk_avail_k"] = avail_k
|
|||
|
|
details["disk_ok"] = avail_k >= 2 * 1024 * 1024
|
|||
|
|
ok = ok and details["disk_ok"]
|
|||
|
|
else:
|
|||
|
|
details["disk_ok"] = False
|
|||
|
|
ok = False
|
|||
|
|
rc, out, _ = ssh_exec(ssh, "ulimit -n")
|
|||
|
|
try:
|
|||
|
|
nofile = int(out.strip())
|
|||
|
|
details["ulimit_nofile"] = nofile
|
|||
|
|
details["ulimit_nofile_ok"] = nofile >= 65535
|
|||
|
|
ok = ok and details["ulimit_nofile_ok"]
|
|||
|
|
except:
|
|||
|
|
details["ulimit_nofile_ok"] = False
|
|||
|
|
ok = False
|
|||
|
|
rc, out, _ = ssh_exec(ssh, "ulimit -u")
|
|||
|
|
try:
|
|||
|
|
nproc = int(out.strip())
|
|||
|
|
details["ulimit_nproc"] = nproc
|
|||
|
|
details["ulimit_nproc_ok"] = nproc >= 4096
|
|||
|
|
ok = ok and details["ulimit_nproc_ok"]
|
|||
|
|
except:
|
|||
|
|
details["ulimit_nproc_ok"] = False
|
|||
|
|
ok = False
|
|||
|
|
return ok, details
|
|||
|
|
|
|||
|
|
|
|||
|
|
def stop_process(ssh, jar_name):
|
|||
|
|
"""优雅停止现有服务：SIGTERM 等待 30s，超时则 SIGKILL"""
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f"ps -ef | grep 'java.*{re.escape(jar_name)}' | grep -v grep | awk '{{print $2}}'")
|
|||
|
|
pids = [p.strip() for p in out.splitlines() if p.strip()]
|
|||
|
|
for pid in pids:
|
|||
|
|
ssh_exec(ssh, f"kill -15 {pid}")
|
|||
|
|
deadline = time.time() + 30
|
|||
|
|
remaining = pids[:]
|
|||
|
|
while time.time() < deadline and remaining:
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f"ps -o pid= -p {' '.join(remaining)}")
|
|||
|
|
alive = [p.strip() for p in out.splitlines() if p.strip()]
|
|||
|
|
remaining = alive
|
|||
|
|
time.sleep(1)
|
|||
|
|
for pid in remaining:
|
|||
|
|
ssh_exec(ssh, f"kill -9 {pid}")
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
def backup_remote(ssh, remote_dir, jar_name, impl_ver):
|
|||
|
|
"""备份当前版本至 /opt/app/backups，并保留最近 5 个版本"""
|
|||
|
|
ssh_exec(ssh, f'mkdir -p "{remote_dir}/backups"')
|
|||
|
|
ts = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
|
|||
|
|
bn = os.path.splitext(jar_name)[0]
|
|||
|
|
ver = impl_ver or "unknown"
|
|||
|
|
backup_name = f"{bn}-{ver}-{ts}.jar"
|
|||
|
|
ssh_exec(ssh, f'if [ -f "{remote_dir}/{jar_name}" ]; then cp -f "{remote_dir}/{jar_name}" "{remote_dir}/backups/{backup_name}"; fi')
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f'ls -1t "{remote_dir}/backups" | grep -E "^{re.escape(bn)}-.*\\.jar$"')
|
|||
|
|
lines = [l.strip() for l in out.splitlines() if l.strip()]
|
|||
|
|
if len(lines) > 5:
|
|||
|
|
to_del = lines[5:]
|
|||
|
|
names = " ".join([f'"{remote_dir}/backups/{x}"' for x in to_del])
|
|||
|
|
ssh_exec(ssh, f"rm -f {names}")
|
|||
|
|
return backup_name
|
|||
|
|
|
|||
|
|
|
|||
|
|
def start_service(ssh, remote_dir, jar_name, jvm_opts):
|
|||
|
|
"""使用 nohup 启动服务并返回 PID 与日志路径"""
|
|||
|
|
log_path = f"{remote_dir}/log.out"
|
|||
|
|
cmd = f'nohup java {jvm_opts} -jar "{remote_dir}/{jar_name}" > "{log_path}" 2>&1 &'
|
|||
|
|
ssh_exec(ssh, cmd)
|
|||
|
|
time.sleep(2)
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f"ps -ef | grep 'java.*{re.escape(jar_name)}' | grep -v grep | awk '{{print $2}}' | head -1")
|
|||
|
|
pid = out.strip() if out.strip() else ""
|
|||
|
|
return pid, log_path
|
|||
|
|
|
|||
|
|
|
|||
|
|
def monitor_logs_no_error(ssh, log_path, seconds):
|
|||
|
|
"""在指定时长内监控启动日志，若出现 ERROR 则失败"""
|
|||
|
|
end = time.time() + seconds
|
|||
|
|
while time.time() < end:
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f'tail -n 200 "{log_path}" || true', print_output=False)
|
|||
|
|
if "ERROR" in out:
|
|||
|
|
return False
|
|||
|
|
time.sleep(2)
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
def detect_crash_within(ssh, jar_name, seconds):
|
|||
|
|
"""监控指定时长，若进程曾启动后又消失，判定为崩溃"""
|
|||
|
|
end = time.time() + seconds
|
|||
|
|
alive_once = False
|
|||
|
|
while time.time() < end:
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f"ps -ef | grep 'java.*{re.escape(jar_name)}' | grep -v grep | awk '{{print $2}}'")
|
|||
|
|
pids = [p.strip() for p in out.splitlines() if p.strip()]
|
|||
|
|
if pids:
|
|||
|
|
alive_once = True
|
|||
|
|
else:
|
|||
|
|
if alive_once:
|
|||
|
|
return True
|
|||
|
|
time.sleep(5)
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def verify_deploy_info(ssh, url, expected_version=None, expected_build_time=None):
|
|||
|
|
"""调用部署验证接口并校验版本与构建时间"""
|
|||
|
|
rc, code_str, _ = ssh_exec(ssh, f'curl -s -o /dev/null -w "%{{http_code}}" "{url}"')
|
|||
|
|
rc2, body, _ = ssh_exec(ssh, f'curl -s "{url}"')
|
|||
|
|
http_code = code_str.strip()
|
|||
|
|
ok_http = http_code == "200"
|
|||
|
|
version = None
|
|||
|
|
build_time = None
|
|||
|
|
try:
|
|||
|
|
data = json.loads(body)
|
|||
|
|
payload = data.get("data") if isinstance(data, dict) and "data" in data else data
|
|||
|
|
if isinstance(payload, dict):
|
|||
|
|
version = payload.get("version")
|
|||
|
|
build_time = payload.get("buildTime")
|
|||
|
|
except:
|
|||
|
|
pass
|
|||
|
|
match_ver = (expected_version is None) or (version == expected_version)
|
|||
|
|
match_bt = (not expected_build_time) or (build_time == expected_build_time)
|
|||
|
|
ok = ok_http and match_ver and match_bt
|
|||
|
|
return ok, {"http_code": http_code, "version": version, "buildTime": build_time}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def rollback(ssh, remote_dir, jar_name, jvm_opts):
|
|||
|
|
"""自动回滚到最近可用备份并重启服务"""
|
|||
|
|
rc, out, _ = ssh_exec(ssh, f'ls -1t "{remote_dir}/backups" | grep -E "^{re.escape(os.path.splitext(jar_name)[0])}-.*\\.jar$" | head -1')
|
|||
|
|
last = out.strip()
|
|||
|
|
if not last:
|
|||
|
|
return False
|
|||
|
|
ssh_exec(ssh, f'cp -f "{remote_dir}/backups/{last}" "{remote_dir}/{jar_name}"')
|
|||
|
|
pid, log_path = start_service(ssh, remote_dir, jar_name, jvm_opts)
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
|
|||
|
|
def send_alert(webhook, payload):
|
|||
|
|
"""向运维告警地址发送 JSON 通知（可选）"""
|
|||
|
|
if not webhook:
|
|||
|
|
return False
|
|||
|
|
data = json.dumps(payload).encode("utf-8")
|
|||
|
|
req = urllib.request.Request(webhook, data=data, headers={"Content-Type": "application/json"})
|
|||
|
|
try:
|
|||
|
|
urllib.request.urlopen(req, timeout=10)
|
|||
|
|
return True
|
|||
|
|
except urllib.error.URLError:
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_bool(s):
|
|||
|
|
"""解析布尔字符串（支持 true/false/1/0/yes/no/on/off）"""
|
|||
|
|
if s is None:
|
|||
|
|
return False
|
|||
|
|
v = str(s).strip().lower()
|
|||
|
|
return v in ("1", "true", "yes", "y", "on")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_env_file(path):
|
|||
|
|
"""加载 .env 配置文件：支持注释与引号包裹的值"""
|
|||
|
|
data = {}
|
|||
|
|
with open(path, "r", encoding="utf-8") as f:
|
|||
|
|
for raw in f:
|
|||
|
|
line = raw.strip()
|
|||
|
|
if not line:
|
|||
|
|
continue
|
|||
|
|
if line.startswith("#") or line.startswith(";"):
|
|||
|
|
continue
|
|||
|
|
if "=" not in line:
|
|||
|
|
continue
|
|||
|
|
k, v = line.split("=", 1)
|
|||
|
|
k = k.strip()
|
|||
|
|
v = v.strip()
|
|||
|
|
if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")):
|
|||
|
|
v = v[1:-1]
|
|||
|
|
data[k] = v
|
|||
|
|
return {
|
|||
|
|
"server": data.get("SERVER"),
|
|||
|
|
"port": int(data.get("PORT")) if data.get("PORT") else None,
|
|||
|
|
"user": data.get("USER"),
|
|||
|
|
"password": data.get("PASSWORD"),
|
|||
|
|
"keyfile": data.get("KEYFILE"),
|
|||
|
|
"fingerprint": data.get("FINGERPRINT"),
|
|||
|
|
"env": data.get("ENV"),
|
|||
|
|
"build_tool": data.get("BUILD_TOOL"),
|
|||
|
|
"project_dir": data.get("PROJECT_DIR"),
|
|||
|
|
"maven_home": data.get("MAVEN_HOME"),
|
|||
|
|
"gradle_home": data.get("GRADLE_HOME"),
|
|||
|
|
"target_jar_name": data.get("TARGET_JAR_NAME"),
|
|||
|
|
"remote_dir": data.get("REMOTE_DIR") or "/opt/app",
|
|||
|
|
"jvm_opts": data.get("JVM_OPTS") or "-Xms512m -Xmx1024m",
|
|||
|
|
"transfer_method": (data.get("TRANSFER_METHOD") or "sftp").lower(),
|
|||
|
|
"transfer_timeout": int(data.get("TRANSFER_TIMEOUT")) if data.get("TRANSFER_TIMEOUT") else 300,
|
|||
|
|
"deploy_info_url": data.get("DEPLOY_INFO_URL") or "http://localhost:8080/deploy/info",
|
|||
|
|
"alert_webhook": data.get("ALERT_WEBHOOK"),
|
|||
|
|
"dry_run": parse_bool(data.get("DRY_RUN")),
|
|||
|
|
"skip_build": parse_bool(data.get("SKIP_BUILD")),
|
|||
|
|
"skip_prechecks": parse_bool(data.get("SKIP_PRECHECKS")),
|
|||
|
|
"precheck_strict": parse_bool(data.get("PRECHECK_STRICT")),
|
|||
|
|
"crash_monitor_seconds": int(data.get("CRASH_MONITOR_SECONDS")) if data.get("CRASH_MONITOR_SECONDS") else 60,
|
|||
|
|
"skip_crash_monitor": parse_bool(data.get("SKIP_CRASH_MONITOR")),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
"""主流程：加载配置 -> 构建 -> 校验 -> 传输 -> 预检 -> 停止 -> 备份 -> 启动 -> 健康 -> 回滚/完成"""
|
|||
|
|
default_env = os.path.join(os.path.dirname(__file__), "deployment.env")
|
|||
|
|
cfg_path = os.getenv("DEPLOY_CONFIG") or default_env
|
|||
|
|
# 仅支持 .env 配置文件，并校验必填字段
|
|||
|
|
if not os.path.isfile(cfg_path):
|
|||
|
|
log_step("config_load", "fail", {"path": cfg_path})
|
|||
|
|
print(f"配置文件未找到：{cfg_path}", file=sys.stderr)
|
|||
|
|
sys.exit(1)
|
|||
|
|
cfg = load_env_file(cfg_path)
|
|||
|
|
required = ["server", "port", "user", "env", "build_tool", "project_dir"]
|
|||
|
|
missing = []
|
|||
|
|
for k in required:
|
|||
|
|
v = cfg.get(k)
|
|||
|
|
if v is None or (isinstance(v, str) and v.strip() == ""):
|
|||
|
|
missing.append(k)
|
|||
|
|
if missing:
|
|||
|
|
log_step("config_validate", "fail", {"missing": missing, "path": cfg_path})
|
|||
|
|
print(f"配置错误：缺少必填项 {', '.join(missing)}；配置文件：{cfg_path}", file=sys.stderr)
|
|||
|
|
sys.exit(1)
|
|||
|
|
print_progress(0, "启动部署", "进行中", f"配置文件：{cfg_path}")
|
|||
|
|
|
|||
|
|
args = SimpleNamespace(
|
|||
|
|
server=cfg["server"],
|
|||
|
|
port=int(cfg["port"]),
|
|||
|
|
user=cfg["user"],
|
|||
|
|
password=cfg.get("password"),
|
|||
|
|
keyfile=cfg.get("keyfile"),
|
|||
|
|
fingerprint=cfg["fingerprint"],
|
|||
|
|
env=cfg["env"],
|
|||
|
|
build_tool=cfg["build_tool"],
|
|||
|
|
project_dir=cfg["project_dir"],
|
|||
|
|
maven_home=cfg.get("maven_home"),
|
|||
|
|
gradle_home=cfg.get("gradle_home"),
|
|||
|
|
target_jar_name=cfg.get("target_jar_name"),
|
|||
|
|
remote_dir=cfg.get("remote_dir", "/opt/app"),
|
|||
|
|
jvm_opts=cfg.get("jvm_opts", "-Xms512m -Xmx1024m"),
|
|||
|
|
deploy_info_url=cfg.get("deploy_info_url", "http://localhost:8080/deploy/info"),
|
|||
|
|
alert_webhook=cfg.get("alert_webhook"),
|
|||
|
|
dry_run=bool(cfg.get("dry_run", False)),
|
|||
|
|
skip_build=bool(cfg.get("skip_build", False)),
|
|||
|
|
transfer_method=cfg.get("transfer_method", "sftp"),
|
|||
|
|
transfer_timeout=int(cfg.get("transfer_timeout", 300)),
|
|||
|
|
skip_prechecks=bool(cfg.get("skip_prechecks", False)),
|
|||
|
|
precheck_strict=bool(cfg.get("precheck_strict", False)),
|
|||
|
|
crash_monitor_seconds=int(cfg.get("crash_monitor_seconds", 60)),
|
|||
|
|
skip_crash_monitor=bool(cfg.get("skip_crash_monitor", False)),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
env_info = {
|
|||
|
|
"server": args.server,
|
|||
|
|
"port": args.port,
|
|||
|
|
"env": args.env,
|
|||
|
|
"build_tool": args.build_tool,
|
|||
|
|
"project_dir": args.project_dir,
|
|||
|
|
"remote_dir": args.remote_dir,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
if args.dry_run:
|
|||
|
|
log_step("start", "dry_run", {"config_path": cfg_path}, env=env_info)
|
|||
|
|
print_progress(5, "加载配置", "完成")
|
|||
|
|
else:
|
|||
|
|
log_step("start", "running", {"config_path": cfg_path}, env=env_info)
|
|||
|
|
print_progress(5, "加载配置", "完成")
|
|||
|
|
|
|||
|
|
if not args.skip_build:
|
|||
|
|
print_progress(10, "开始构建", "进行中", args.build_tool)
|
|||
|
|
if args.build_tool == "maven":
|
|||
|
|
rc, out, err = maven_build(args.project_dir, args.maven_home or "")
|
|||
|
|
else:
|
|||
|
|
rc, out, err = gradle_build(args.project_dir, args.gradle_home)
|
|||
|
|
log_step("build", "ok" if rc == 0 else "fail", {"stdout": out[-1000:], "stderr": err[-1000:]}, env=env_info)
|
|||
|
|
print_progress(20, "构建完成", "成功" if rc == 0 else "失败")
|
|||
|
|
if rc != 0 and not args.dry_run:
|
|||
|
|
sys.exit(1)
|
|||
|
|
else:
|
|||
|
|
print_progress(10, "跳过构建", "已跳过")
|
|||
|
|
|
|||
|
|
jar_path = find_jar(args.project_dir, args.build_tool, args.target_jar_name)
|
|||
|
|
if not jar_path or not os.path.isfile(jar_path):
|
|||
|
|
log_step("find_jar", "fail", {"path": jar_path}, env=env_info)
|
|||
|
|
print_progress(25, "查找构建产物", "失败")
|
|||
|
|
if not args.dry_run:
|
|||
|
|
sys.exit(1)
|
|||
|
|
else:
|
|||
|
|
log_step("find_jar", "ok", {"path": jar_path}, env=env_info)
|
|||
|
|
print_progress(25, "查找构建产物", "成功", os.path.basename(jar_path))
|
|||
|
|
|
|||
|
|
if jar_path and os.path.isfile(jar_path):
|
|||
|
|
ok, missing = verify_jar_integrity(jar_path)
|
|||
|
|
manifest_info = parse_manifest_info(jar_path)
|
|||
|
|
log_step("jar_integrity", "ok" if ok else "fail", {"manifest": manifest_info, "missing": missing}, env=env_info)
|
|||
|
|
print_progress(35, "校验产物完整性", "成功" if ok else f"失败(缺失: {', '.join(missing)})")
|
|||
|
|
if not ok and not args.dry_run:
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
if args.dry_run:
|
|||
|
|
log_step("fingerprint_check", "skip", env=env_info)
|
|||
|
|
log_step("transfer", "skip", env=env_info)
|
|||
|
|
log_step("pre_checks", "skip", env=env_info)
|
|||
|
|
log_step("stop_process", "skip", env=env_info)
|
|||
|
|
log_step("backup", "skip", env=env_info)
|
|||
|
|
log_step("start_service", "skip", env=env_info)
|
|||
|
|
log_step("monitor_crash", "skip", env=env_info)
|
|||
|
|
print_progress(100, "完成", "DRY-RUN")
|
|||
|
|
log_step("done", "ok", env=env_info)
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
if args.fingerprint:
|
|||
|
|
print_progress(40, "校验服务器指纹", "进行中")
|
|||
|
|
ok_fp, actual_fp = ensure_fingerprint(args.server, args.port, args.user, args.password, args.keyfile, args.fingerprint)
|
|||
|
|
log_step("fingerprint_check", "ok" if ok_fp else "fail", {"actual": actual_fp}, env=env_info)
|
|||
|
|
print_progress(45, "校验服务器指纹", "成功" if ok_fp else "失败")
|
|||
|
|
if not ok_fp:
|
|||
|
|
print(f"服务器指纹不匹配：期望 {args.fingerprint}，实际 {actual_fp}", file=sys.stderr)
|
|||
|
|
sys.exit(1)
|
|||
|
|
else:
|
|||
|
|
log_step("fingerprint_check", "skip", {"reason": "no_fingerprint_provided"}, env=env_info)
|
|||
|
|
print_progress(40, "校验服务器指纹", "已跳过")
|
|||
|
|
|
|||
|
|
print_progress(50, "连接服务器", "进行中")
|
|||
|
|
ssh = connect_ssh(args.server, args.port, args.user, args.password, args.keyfile)
|
|||
|
|
ssh.exec_command(f'mkdir -p "{args.remote_dir}"')
|
|||
|
|
print_progress(55, "连接服务器", "成功")
|
|||
|
|
|
|||
|
|
local_md5 = md5_file(jar_path)
|
|||
|
|
print_progress(60, "传输文件", "进行中", os.path.basename(jar_path))
|
|||
|
|
if args.transfer_method == "scp":
|
|||
|
|
rc, out, err = scp_transfer(
|
|||
|
|
jar_path,
|
|||
|
|
args.server,
|
|||
|
|
args.port,
|
|||
|
|
args.user,
|
|||
|
|
f'{args.remote_dir}/{os.path.basename(jar_path)}',
|
|||
|
|
timeout=args.transfer_timeout,
|
|||
|
|
)
|
|||
|
|
else:
|
|||
|
|
rc, out, err = sftp_transfer(
|
|||
|
|
ssh,
|
|||
|
|
jar_path,
|
|||
|
|
f'{args.remote_dir}/{os.path.basename(jar_path)}',
|
|||
|
|
)
|
|||
|
|
log_step("transfer", "ok" if rc == 0 else "fail", {"stdout": out[-1000:], "stderr": err[-1000:]}, env=env_info)
|
|||
|
|
print_progress(65, "传输文件", "成功" if rc == 0 else "失败")
|
|||
|
|
if rc != 0:
|
|||
|
|
ssh.close()
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
remote_hash, algo = remote_md5(ssh, f'{args.remote_dir}/{os.path.basename(jar_path)}')
|
|||
|
|
match = remote_hash == local_md5 if algo == "md5" and remote_hash else True
|
|||
|
|
log_step("md5_verify", "ok" if match else "fail", {"local_md5": local_md5, "remote_hash": remote_hash, "algo": algo}, env=env_info)
|
|||
|
|
print_progress(70, "校验文件摘要", "成功" if match else "失败")
|
|||
|
|
if not match:
|
|||
|
|
ssh.close()
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
if args.skip_prechecks:
|
|||
|
|
log_step("pre_checks", "skip", env=env_info)
|
|||
|
|
print_progress(75, "预部署检查", "已跳过")
|
|||
|
|
else:
|
|||
|
|
checks_ok, check_details = pre_checks(ssh, args.remote_dir)
|
|||
|
|
status = "ok" if checks_ok else ("fail" if args.precheck_strict else "warn")
|
|||
|
|
log_step("pre_checks", status, check_details, env=env_info)
|
|||
|
|
print_progress(75, "预部署检查", "成功" if checks_ok else ("警告" if not args.precheck_strict else "失败"))
|
|||
|
|
if not checks_ok and args.precheck_strict:
|
|||
|
|
ssh.close()
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
jar_name = os.path.basename(jar_path)
|
|||
|
|
print_progress(78, "停止现有服务", "进行中")
|
|||
|
|
stop_process(ssh, jar_name)
|
|||
|
|
log_step("stop_process", "ok", env=env_info)
|
|||
|
|
print_progress(80, "停止现有服务", "完成")
|
|||
|
|
|
|||
|
|
backup_name = backup_remote(ssh, args.remote_dir, jar_name, parse_manifest_info(jar_path).get("Implementation-Version"))
|
|||
|
|
log_step("backup", "ok", {"backup": backup_name}, env=env_info)
|
|||
|
|
print_progress(82, "备份旧版本", "完成", backup_name)
|
|||
|
|
|
|||
|
|
pid, log_path = start_service(ssh, args.remote_dir, jar_name, args.jvm_opts)
|
|||
|
|
log_step("start_service", "ok" if pid else "fail", {"pid": pid, "log": log_path}, env=env_info)
|
|||
|
|
print_progress(85, "启动服务", "成功" if pid else "失败", f"pid={pid}")
|
|||
|
|
if not pid:
|
|||
|
|
ssh.close()
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
log_step("startup_logs", "skip", env=env_info)
|
|||
|
|
print_progress(90, "监控启动日志", "已跳过")
|
|||
|
|
|
|||
|
|
last_metrics = {}
|
|||
|
|
|
|||
|
|
if args.skip_crash_monitor:
|
|||
|
|
log_step("monitor_crash", "skip", env=env_info)
|
|||
|
|
print_progress(97, f"崩溃监控({args.crash_monitor_seconds}秒)", "已跳过")
|
|||
|
|
else:
|
|||
|
|
crashed = detect_crash_within(ssh, jar_name, args.crash_monitor_seconds)
|
|||
|
|
log_step("monitor_crash", "fail" if crashed else "ok", env=env_info)
|
|||
|
|
print_progress(97, f"崩溃监控({args.crash_monitor_seconds}秒)", "失败" if crashed else "通过")
|
|||
|
|
|
|||
|
|
ver_expected = parse_manifest_info(jar_path).get("Implementation-Version")
|
|||
|
|
bt_expected = parse_manifest_info(jar_path).get("Build-Time")
|
|||
|
|
print_progress(98, "验证部署接口", "进行中")
|
|||
|
|
ok_deploy, deploy_info = verify_deploy_info(ssh, args.deploy_info_url, ver_expected, bt_expected)
|
|||
|
|
log_step("deploy_info_check", "ok" if ok_deploy else "fail", {"expect_version": ver_expected, "expect_build_time": bt_expected, "response": deploy_info}, env=env_info)
|
|||
|
|
print_progress(98, "验证部署接口", "成功" if ok_deploy else "失败")
|
|||
|
|
|
|||
|
|
if crashed or not ok_deploy:
|
|||
|
|
print_progress(98, "触发回滚", "进行中")
|
|||
|
|
rb_ok = rollback(ssh, args.remote_dir, jar_name, args.jvm_opts)
|
|||
|
|
log_step("rollback", "ok" if rb_ok else "fail", env=env_info)
|
|||
|
|
print_progress(99, "回滚完成", "成功" if rb_ok else "失败")
|
|||
|
|
send_alert(args.alert_webhook, {"event": "rollback", "server": args.server, "env": args.env, "reason": "deploy_info_fail_or_crash"})
|
|||
|
|
ssh.close()
|
|||
|
|
if not rb_ok:
|
|||
|
|
sys.exit(1)
|
|||
|
|
sys.exit(0)
|
|||
|
|
|
|||
|
|
log_step("done", "ok", {"metrics": last_metrics}, env=env_info)
|
|||
|
|
print_progress(100, "部署完成", "成功")
|
|||
|
|
ssh.close()
|
|||
|
|
except Exception as e:
|
|||
|
|
import traceback
|
|||
|
|
log_step("exception", "fail", {"error": str(e), "trace": traceback.format_exc()}, env=env_info)
|
|||
|
|
print_progress(100, "部署失败", "异常", str(e))
|
|||
|
|
print(traceback.format_exc(), file=sys.stderr)
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|