Russian narration for the Enshrouded notes: mod 0.4.0, installer, generation pipeline
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
"""Russian narration for the Enshrouded notes: extraction, TTS, automatic QC, EML mod."""
|
||||
__version__ = "0.4.0"
|
||||
@@ -0,0 +1,59 @@
|
||||
"""RUAccent stresses for the text of the voice.
|
||||
|
||||
plus_marks(): '+' before the stressed vowel, carried onto the ORIGINAL text word by word. RUAccent is
|
||||
fed lowercase (it is worse on Title-case) and drops spaces before "(", so a char-by-char walk drifts and
|
||||
garbles the tail ("в годод", "))"); here only the n-th Cyrillic word of its output gives its '+' (and ё)
|
||||
to the n-th word of the source, punctuation and spacing stay the source's. If the word counts differ,
|
||||
the text is accented clause by clause.
|
||||
acute_marks(): '+' before a vowel -> the vowel followed by U+0301, the input of the ru-stress model.
|
||||
"""
|
||||
import re
|
||||
|
||||
VOWELS = "аеёиоуыэюяАЕЁИОУЫЭЮЯ"
|
||||
W = re.compile(r"[А-Яа-яЁё]+(?:-[А-Яа-яЁё]+)*")
|
||||
WP = re.compile(r"[А-Яа-яЁё+]+(?:-[А-Яа-яЁё+]+)*")
|
||||
_acc = None
|
||||
|
||||
|
||||
def accentizer():
|
||||
global _acc
|
||||
if _acc is None:
|
||||
from ruaccent import RUAccent
|
||||
_acc = RUAccent()
|
||||
_acc.load(omograph_model_size="turbo3.1", use_dictionary=True, tiny_mode=False)
|
||||
return _acc
|
||||
|
||||
|
||||
def _transfer(orig, acc):
|
||||
out, i = [], 0
|
||||
for ch in acc:
|
||||
if ch == "+":
|
||||
out.append("+"); continue
|
||||
o = orig[i]
|
||||
out.append(("Ё" if o == "Е" else "ё") if ch.lower() == "ё" and o in "еЕ" else o)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _plus(text):
|
||||
acc = accentizer().process_all(text.lower())
|
||||
src, dst = list(W.finditer(text)), WP.findall(acc)
|
||||
if len(src) != len(dst) or any(len(a.group(0)) != len(b.replace("+", "")) for a, b in zip(src, dst)):
|
||||
return None
|
||||
out, last = [], 0
|
||||
for a, b in zip(src, dst):
|
||||
out += [text[last:a.start()], _transfer(a.group(0), b)]
|
||||
last = a.end()
|
||||
return "".join(out) + text[last:]
|
||||
|
||||
|
||||
def plus_marks(text: str) -> str:
|
||||
r = _plus(text)
|
||||
if r is not None:
|
||||
return r
|
||||
parts = re.split(r"([,.;:!?()«»…]+\s*)", text)
|
||||
return "".join(p if i % 2 or not p.strip() else (_plus(p) or p) for i, p in enumerate(parts))
|
||||
|
||||
|
||||
def acute_marks(plus_text: str) -> str:
|
||||
return re.sub(r"\+([" + VOWELS + "])", lambda m: m.group(1) + "́", plus_text).replace("+", "")
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Step 7: join every note from its picked takes and encode it for the game (work/qoa, work/wav48).
|
||||
|
||||
- each take trimmed to its speech (top_db 40), 60 ms kept at both ends, 8 ms fades;
|
||||
- pauses as the EN narrator makes them: PAUSE["s"] inside a paragraph, PAUSE["l"] after a line break,
|
||||
PAUSE["p"] between paragraphs (EN medians: phrases ~0.6 s, paragraphs ~0.9 s);
|
||||
- 24 -> 48 kHz with soxr VHQ; loudness: active-speech RMS matched to the EN original, soft limiter
|
||||
at -1 dBFS; 100 ms of silence at both ends; frameless QOA.
|
||||
The QOA encoder is pure Python: this step is CPU-bound (assemble.workers processes).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from multiprocessing import Pool
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
from .config import load
|
||||
from .qoa import encode
|
||||
|
||||
PAUSE = {"s": 0.45, "l": 0.70, "p": 0.90}
|
||||
CFG = None
|
||||
|
||||
|
||||
def active_rms(x):
|
||||
v = x[::5]
|
||||
v = v[np.abs(v) > 200 / 32768]
|
||||
return float(np.sqrt(np.mean(v * v))) if len(v) else 1e-3
|
||||
|
||||
|
||||
def trim(y, sr):
|
||||
iv = librosa.effects.split(y, top_db=40, frame_length=1024, hop_length=240)
|
||||
if not len(iv):
|
||||
return y
|
||||
y = y[max(0, iv[0][0] - int(0.06 * sr)): min(len(y), iv[-1][1] + int(0.06 * sr))].copy()
|
||||
f = int(0.008 * sr)
|
||||
y[:f] *= np.linspace(0, 1, f)
|
||||
y[-f:] *= np.linspace(1, 0, f)
|
||||
return y
|
||||
|
||||
|
||||
def job(args):
|
||||
n, segs, pick = args
|
||||
cfg = CFG or load()
|
||||
meta = cfg.w("qoa", f"{n['k']}.json")
|
||||
if meta.exists():
|
||||
return
|
||||
parts = []
|
||||
for s in segs:
|
||||
y, sr = sf.read(cfg.w("takes", f"{s['take']}_s{pick[s['id']]}.wav"), dtype="float32")
|
||||
if parts:
|
||||
parts.append(np.zeros(int(PAUSE.get(s["join"], 0.45) * sr), np.float32))
|
||||
parts.append(trim(y, sr))
|
||||
y = librosa.resample(np.concatenate(parts), orig_sr=24000, target_sr=48000, res_type="soxr_vhq")
|
||||
en, _ = sf.read(cfg.w("en24", f"{n['k']}.wav"), dtype="float32")
|
||||
gain = active_rms(en) / active_rms(y)
|
||||
y = y * gain
|
||||
lim = 10 ** (-1 / 20)
|
||||
knee = 0.7 * lim
|
||||
over = np.abs(y) > knee
|
||||
y[over] = np.sign(y[over]) * (knee + (lim - knee) * np.tanh((np.abs(y[over]) - knee) / (lim - knee)))
|
||||
pad = np.zeros(4800, np.float32)
|
||||
y = np.concatenate([pad, y, pad])
|
||||
sf.write(cfg.w("wav48", f"{n['k']}.wav"), y, 48000, subtype="PCM_16")
|
||||
pcm = np.clip(np.round(y * 32767), -32768, 32767).astype(np.int32).tolist()
|
||||
cfg.w("qoa", f"{n['sound']}.qoa").write_bytes(encode(pcm))
|
||||
json.dump(dict(k=n["k"], name=n["name"], vo=n["vo"], sound=n["sound"], frames=len(pcm),
|
||||
seconds=len(pcm) / 48000, gain=round(gain, 3), segments=len(segs)), open(meta, "w"))
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
notes = json.load(open(cfg.w("notes.json"), encoding="utf-8"))
|
||||
segs = json.load(open(cfg.w("segs.json"), encoding="utf-8"))
|
||||
pick = json.load(open(cfg.w("pick.json")))
|
||||
by = {}
|
||||
for s in segs:
|
||||
by.setdefault(s["k"], []).append(s)
|
||||
notes = [n for n in notes if n["k"] in by] # segments may cover a subset (ENSVOICE_NOTES)
|
||||
if "--force" in sys.argv: # re-encode every note (after a new pick)
|
||||
for f in cfg.w("qoa", "x").parent.glob("n*.json"):
|
||||
f.unlink()
|
||||
else: # re-encode the notes whose picked takes changed
|
||||
for n in notes:
|
||||
m = cfg.w("qoa", f"{n['k']}.json")
|
||||
if m.exists() and json.load(open(m)).get("takes") != [f"{s['take']}_s{pick.get(s['id'])}" for s in by[n["k"]]]:
|
||||
m.unlink()
|
||||
jobs = [(n, by[n["k"]], pick) for n in notes if not cfg.w("qoa", f"{n['k']}.json").exists()]
|
||||
print("notes to encode", len(jobs), flush=True)
|
||||
with Pool(int(cfg.get("assemble", "workers", max(1, (os.cpu_count() or 2) - 1)))) as p:
|
||||
for i, _ in enumerate(p.imap_unordered(job, jobs, chunksize=1)):
|
||||
if i % 25 == 0:
|
||||
print(f"{i + 1}/{len(jobs)}", flush=True)
|
||||
for n in notes: # remember which takes each note was built from
|
||||
m = cfg.w("qoa", f"{n['k']}.json")
|
||||
d = json.load(open(m))
|
||||
d["takes"] = [f"{s['take']}_s{pick[s['id']]}" for s in by[n["k"]]]
|
||||
json.dump(d, open(m, "w"))
|
||||
total = sum(json.load(open(cfg.w("qoa", f"{n['k']}.json")))["seconds"] for n in notes)
|
||||
print("ASSEMBLED", len(notes), "notes,", round(total / 60, 1), "minutes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Settings from config.toml (or the file named by ENSVOICE_CONFIG) and the work-directory layout."""
|
||||
import os
|
||||
import pathlib
|
||||
import tomllib
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class Config:
|
||||
def __init__(self, path=None):
|
||||
path = pathlib.Path(path or os.environ.get("ENSVOICE_CONFIG") or ROOT / "config.toml")
|
||||
if not path.exists():
|
||||
raise SystemExit(f"no config: {path} (copy config.example.toml to config.toml and edit it)")
|
||||
self.raw = tomllib.loads(path.read_text(encoding="utf-8"))
|
||||
p = self.raw["paths"]
|
||||
self.game = pathlib.Path(p["game"])
|
||||
self.work = pathlib.Path(p["work"])
|
||||
self.tools = pathlib.Path(p.get("tools", ROOT / "tools"))
|
||||
self.work.mkdir(parents=True, exist_ok=True)
|
||||
# ENSVOICE_NOTES=n016,n238 limits segments (and so everything after it) to these notes
|
||||
self.only = {x.strip() for x in os.environ.get("ENSVOICE_NOTES", "").split(",") if x.strip()}
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.raw[key]
|
||||
|
||||
def get(self, section, key, default=None):
|
||||
return self.raw.get(section, {}).get(key, default)
|
||||
|
||||
# work-directory layout (everything derived from the game stays here, never in the repo)
|
||||
def w(self, *parts) -> pathlib.Path:
|
||||
p = self.work.joinpath(*parts)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def load(path=None) -> Config:
|
||||
return Config(path)
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Step 1: from the installed game to work/notes.json + the EN originals as 24 kHz WAV.
|
||||
|
||||
- kfc-parser unpacks the descriptors we need (voiceover, sound, localisation, journal);
|
||||
- localisation tags of EN and RU are read straight from the .dat content (24-byte tag records);
|
||||
- every journal entry with a voiceover joins its EN sound and its EN/RU text;
|
||||
- the RU text is turned into the text for the voice (tags off, dashes to commas, numbers in words);
|
||||
- the EN audio (frameless QOA, 48 kHz) is decoded to work/en24/<k>.wav with its loudness.
|
||||
The game files are only read. Keys n000..n524 follow the EN sound names sorted.
|
||||
|
||||
A game with this mod (or any EML mod) applied has a PATCHED enshrouded.kfc: its index then points to the
|
||||
Russian takes. EML appends new content and keeps the original index as enshrouded.kfc.bak, so the
|
||||
originals are read through work/orig: a copy of the .bak as enshrouded.kfc plus hard links to the .dat
|
||||
and .kfc_resources files (the work folder must then be on the game's drive; otherwise run
|
||||
`tools/emm.exe restore` first — the mod is applied again at the next game launch).
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import wave
|
||||
from multiprocessing import Pool
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .config import load
|
||||
from .kfc import KFC
|
||||
from .qoa import decode
|
||||
from .ru_num import normalize
|
||||
|
||||
TYPES = ["VoiceoverResource", "SoundContainerResource", "SoundResource",
|
||||
"LocaTagCollectionResource", "JournalRegistryResource"]
|
||||
|
||||
|
||||
def source(cfg):
|
||||
"""The game folder with the ORIGINAL index (see the module docstring)."""
|
||||
bak = cfg.game / "enshrouded.kfc.bak"
|
||||
if not bak.exists():
|
||||
return cfg.game
|
||||
orig = cfg.w("orig", "x").parent
|
||||
shutil.copyfile(bak, orig / "enshrouded.kfc")
|
||||
for f in list(cfg.game.glob("enshrouded_*.dat")) + [cfg.game / "enshrouded.kfc_resources"]:
|
||||
dst = orig / f.name
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
try:
|
||||
os.link(f, dst)
|
||||
except OSError:
|
||||
raise SystemExit(f"the game has EML mods applied and {orig} cannot hard-link {f.name} "
|
||||
f"(another drive?): put paths.work on the game's drive, or run "
|
||||
f"'{cfg.tools / 'emm.exe'} restore -g {cfg.game}' and extract again")
|
||||
print("reading the original index from enshrouded.kfc.bak (the game has EML mods applied)")
|
||||
return orig
|
||||
|
||||
|
||||
def unpack(cfg, src):
|
||||
out = cfg.w("unpack")
|
||||
if all(glob.glob(str(out / t / "*.json")) for t in TYPES):
|
||||
print("unpack: already done"); return out
|
||||
exe = cfg.tools / "kfc-parser.exe"
|
||||
flt = ",".join(f"tkeen::{t}" for t in TYPES)
|
||||
subprocess.run([str(exe), "unpack", "-g", str(src), "-o", str(out), "-f", flt], check=True)
|
||||
return out
|
||||
|
||||
|
||||
def by_guid(d, t):
|
||||
return {os.path.basename(f).split("_")[0]: f for f in glob.glob(str(d / t / "*.json"))}
|
||||
|
||||
|
||||
def loca(cfg, kfc, d):
|
||||
res = json.load(open(glob.glob(str(d / "LocaTagCollectionResource" / "*.json"))[0], encoding="utf-8"))
|
||||
langs = {l["language"]: l["data"] for l in res["languages"]}
|
||||
out = {}
|
||||
for name in ("En_Us", "Ru_Ru"):
|
||||
data = kfc.read_hash(langs[name]["hash"])
|
||||
off, cnt = struct.unpack_from("<II", data, 0)
|
||||
tags = {}
|
||||
for i in range(cnt):
|
||||
p = off + i * 24
|
||||
tid, so, sl = struct.unpack_from("<III", data, p)
|
||||
tags[str(tid)] = data[p + 4 + so:p + 4 + so + sl].decode("utf-8", "replace")
|
||||
out[name] = tags
|
||||
print("loca", name, len(tags), "tags")
|
||||
return out
|
||||
|
||||
|
||||
def voice_index(d):
|
||||
sc, sr = by_guid(d, "SoundContainerResource"), by_guid(d, "SoundResource")
|
||||
rows = []
|
||||
for f in glob.glob(str(d / "VoiceoverResource" / "*.json")):
|
||||
vo = json.load(open(f, encoding="utf-8"))
|
||||
en = vo["languages"][0] # fixed slots EN/DE/FR/ES/ZH
|
||||
if en not in sc:
|
||||
continue
|
||||
for e in json.load(open(sc[en], encoding="utf-8"))["entries"]:
|
||||
s = e["$value"].get("sound")
|
||||
if s in sr:
|
||||
r = json.load(open(sr[s], encoding="utf-8"))
|
||||
rows.append(dict(vo=vo["$guid"], sound=s, name=r["debugName"],
|
||||
sec=r["frameCount"] / r["framesPerSecond"], h=r["streamedData"]["hash"]))
|
||||
return rows
|
||||
|
||||
|
||||
def journal(d):
|
||||
j = json.load(open(glob.glob(str(d / "JournalRegistryResource" / "*.json"))[0], encoding="utf-8"))
|
||||
m = {}
|
||||
|
||||
def walk(o):
|
||||
if isinstance(o, dict):
|
||||
if o.get("voiceover") and "text" in o:
|
||||
m[o["voiceover"]] = (o["name"]["value"], o["text"]["value"])
|
||||
for v in o.values():
|
||||
walk(v)
|
||||
elif isinstance(o, list):
|
||||
for v in o:
|
||||
walk(v)
|
||||
walk(j)
|
||||
return m
|
||||
|
||||
|
||||
def screen(t):
|
||||
return re.sub(r"<[^>]+>", "", t or "")
|
||||
|
||||
|
||||
YO = {"еще": "ещё", "все равно": "всё равно", "трясется": "трясётся", "придется": "придётся", "намек": "намёк",
|
||||
"идет": "идёт", "ее ": "её ", "черный": "чёрный", "темн": "тёмн", "зеленый": "зелёный", "живет": "живёт",
|
||||
"дает": "даёт", "берет": "берёт"}
|
||||
|
||||
|
||||
def voice_text(t):
|
||||
"""The RU screen text -> the text for the voice (the subtitles keep the screen text)."""
|
||||
t = screen(t)
|
||||
t = re.sub(r"[ \t]+\n", "\n", t); t = re.sub(r"\n[ \t]+", "\n", t)
|
||||
t = re.sub(r"^\s*[—–-]\s*", "", t, flags=re.M) # list dashes at a line start
|
||||
t = re.sub(r"\s+[—–]\s+", ", ", t) # a dash makes a long pause: a comma
|
||||
t = re.sub(r"(\w)-(то|ка|нибудь|либо)\b", r"\1 \2", t)
|
||||
for a, b in YO.items():
|
||||
t = re.sub(r"\b" + a, b, t)
|
||||
t = re.sub(r"\n{3,}", "\n\n", t).strip()
|
||||
t = normalize(t).strip()
|
||||
return re.sub(r"\.{4,}", "...", t)
|
||||
|
||||
|
||||
_FIR = None
|
||||
|
||||
|
||||
def _en_job(args):
|
||||
game, k, r, out = args
|
||||
global _FIR
|
||||
if _FIR is None:
|
||||
n = np.arange(-48, 49)
|
||||
_FIR = np.sinc(n * 0.46) * np.hamming(len(n)); _FIR /= _FIR.sum()
|
||||
s = np.asarray(decode(KFC(game).read_hash(r["h"]), int(r["en_sec"] * 48000)), np.float64)
|
||||
v = s[::5]; v = v[np.abs(v) > 200]
|
||||
rms = float(np.sqrt(np.mean(v * v))) if len(v) else 1000.0
|
||||
y = np.convolve(s, _FIR, mode="same")[::2]
|
||||
with wave.open(out, "wb") as w:
|
||||
w.setnchannels(1); w.setsampwidth(2); w.setframerate(24000)
|
||||
w.writeframes(np.clip(y, -32768, 32767).astype("<i2").tobytes())
|
||||
return k, rms
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
src = source(cfg)
|
||||
d = unpack(cfg, src)
|
||||
kfc = KFC(src)
|
||||
L = loca(cfg, kfc, d)
|
||||
rows = voice_index(d)
|
||||
byvo = {}
|
||||
for r in rows:
|
||||
byvo.setdefault(r["vo"], []).append(r)
|
||||
notes = []
|
||||
for vo, (nm, tx) in journal(d).items():
|
||||
for r in byvo.get(vo, []):
|
||||
ru = L["Ru_Ru"].get(str(tx))
|
||||
if not ru:
|
||||
continue
|
||||
notes.append(dict(name=r["name"], vo=vo, sound=r["sound"], en_sec=r["sec"], h=r["h"],
|
||||
title_en=screen(L["En_Us"].get(str(nm))), en=L["En_Us"].get(str(tx)),
|
||||
ru_screen=screen(ru), text=voice_text(ru)))
|
||||
notes.sort(key=lambda x: x["name"])
|
||||
for i, n in enumerate(notes):
|
||||
n["k"] = f"n{i:03d}"
|
||||
os.makedirs(cfg.w("en24", "x").parent, exist_ok=True)
|
||||
jobs = [(str(src), n["k"], n, str(cfg.w("en24", f"{n['k']}.wav"))) for n in notes
|
||||
if not os.path.exists(cfg.w("en24", f"{n['k']}.wav"))]
|
||||
rms = {}
|
||||
if os.path.exists(cfg.w("en_rms.json")):
|
||||
rms = json.load(open(cfg.w("en_rms.json")))
|
||||
with Pool(max(1, (os.cpu_count() or 2) - 1)) as p:
|
||||
for k, v in p.imap_unordered(_en_job, jobs, chunksize=1):
|
||||
rms[k] = v
|
||||
json.dump(rms, open(cfg.w("en_rms.json"), "w"))
|
||||
for n in notes:
|
||||
n["en_rms"] = rms.get(n["k"])
|
||||
json.dump(notes, open(cfg.w("notes.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||||
print("notes", len(notes), "EN minutes", round(sum(n["en_sec"] for n in notes) / 60),
|
||||
"RU chars", sum(len(n["text"]) for n in notes))
|
||||
if len(notes) != 525:
|
||||
print("WARNING: expected 525 voiced notes (game 1.0, 2026-10); a game update may have changed them",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Minimal reader of Enshrouded's KFC4 container: raw content blobs by their content key.
|
||||
|
||||
enshrouded.kfc header: 17 KFCLocation entries (u32 offset relative to the entry, u32 count) at 16 + i*8.
|
||||
Location 1 = .dat containers, 7 = content keys (size, hash0, hash1, hash2), 8 = values
|
||||
(offset u32, flags u16, container index u16, 8 bytes pad). Blobs live in enshrouded_NNN.dat.
|
||||
"""
|
||||
import pathlib
|
||||
import struct
|
||||
|
||||
|
||||
class KFC:
|
||||
def __init__(self, game):
|
||||
self.game = pathlib.Path(game)
|
||||
b = (self.game / "enshrouded.kfc").read_bytes()
|
||||
assert b[:4] == b"KFC4", "not a KFC4 file"
|
||||
locs = []
|
||||
for i in range(17):
|
||||
pos = 16 + i * 8
|
||||
off, cnt = struct.unpack_from("<II", b, pos)
|
||||
locs.append((pos + off if off else 0, cnt))
|
||||
ko, kc = locs[7]
|
||||
vo, vc = locs[8]
|
||||
assert kc == vc
|
||||
self.content = {}
|
||||
for i in range(kc):
|
||||
key = struct.unpack_from("<IIII", b, ko + i * 16)
|
||||
off, flags, ci = struct.unpack_from("<IHH", b, vo + i * 16)
|
||||
self.content[key] = (off, flags, ci)
|
||||
|
||||
def read(self, size, hash0, hash1, hash2):
|
||||
e = self.content.get((size, hash0, hash1, hash2))
|
||||
if not e:
|
||||
return None
|
||||
off, _, ci = e
|
||||
with open(self.game / f"enshrouded_{ci:03d}.dat", "rb") as f:
|
||||
f.seek(off)
|
||||
return f.read(size)
|
||||
|
||||
def read_hash(self, h: dict):
|
||||
"""h: a descriptor's content hash {"size", "hash0", "hash1", "hash2"}."""
|
||||
return self.read(h["size"], h["hash0"], h["hash1"], h["hash2"])
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Voice-only text fixes for the notes: pre() on the plain text, post() on the RUAccent '+' text.
|
||||
|
||||
The screen text and the subtitles never see these. Add a word here, re-run `segments`, and only the
|
||||
segments whose text changed are rendered and checked again.
|
||||
|
||||
- spelling: Latin look-alikes, stray symbols, apostrophe names (Кра'Тау -> Кра Тау);
|
||||
- STRESS: game names by their English stress, keyed by a stem regex; the value is the index of the
|
||||
stressed vowel inside the word (0 = first). Existing '+' in a matched word is replaced;
|
||||
- words with two '+' in one part (Фир+аж+а) keep the mark the lexicon gives or else the last one.
|
||||
"""
|
||||
import re
|
||||
|
||||
VOW = "аеёиоуыэюяАЕЁИОУЫЭЮЯ"
|
||||
|
||||
PRE = [ # plain text, before RUAccent
|
||||
(r"(?<=[А-Яа-яЁё ])c(?=[ А-Яа-яЁё])", "с"), (r"(?<=[А-Яа-яЁё ])C(?=[ а-яё])", "С"),
|
||||
(r"»>", "»"), (r">", ""),
|
||||
(r"Пыш'Ка", "Пышка"), (r"Гр[уy]ул", "Грул"),
|
||||
# abbreviations and the numbers around them (the only cases in the notes)
|
||||
(r"ок\. одна тысяча двести мм", "около тысячи двухсот миллиметров"), (r"ок\. семьсот мм", "около семисот миллиметров"),
|
||||
(r"ок\. триста пятьдесят мм", "около трёхсот пятидесяти миллиметров"), (r"ок\. три тысячи мм", "около трёх тысяч миллиметров"),
|
||||
(r"от пятьсот мм", "от пятисот миллиметров"), (r"до две тысячи мм", "до двух тысяч миллиметров"),
|
||||
(r"свыше одна тысяча лет", "свыше тысячи лет"), (r"\bиз сто\b", "из ста"), (r"\bот сто\b", "от ста"),
|
||||
(r"\bмм\b", "миллиметров"), (r"\bв т\.\s?ч\.:?", "в том числе"), (r"\(см\. ", "(смотри "), (r"\bсм\. ниже", "смотри ниже"),
|
||||
(r"([А-ЯЁа-яё]+)'([А-ЯЁа-яё]+)", lambda m: m.group(1) + " " + m.group(2).lower()
|
||||
if m.group(1).lower().startswith("драк") else m.group(1) + " " + m.group(2)),
|
||||
]
|
||||
|
||||
# stem regex (whole word, case-insensitive, may carry endings) -> stressed vowel index
|
||||
STRESS = {
|
||||
r"воргот\w*": 0, r"пайкмид\w*": 0, r"эмбервэйл\w*": 0, r"римгард\w*": 0, r"мелдор\w*": 0,
|
||||
r"морго": 0, r"лэнсхарт\w*": 0, r"гленвудз": 0, r"констанцио": 1, r"мейерс\w*": 0,
|
||||
r"лэпис\w*": 0, r"хэрриот\w*": 0, r"тау": 0, r"тул": 0, r"раш\w*": 0, r"ках\w*": 0,
|
||||
r"конвэ\w*": 0, r"герит\w*": 0, r"эндрю": 0, r"лахлан\w*": 0, r"бриттлбуш\w*": 0,
|
||||
r"лонгкип\w*": 0, r"аркрайт\w*": 0, r"вилли": 0, r"орвил\w*": 0, r"пышка": 0,
|
||||
r"уайтуинд\w*": 0, r"харпер\w*": 0, r"везт\w*": 0, r"стилуотер": 0, r"голдрок": 0,
|
||||
r"арлингтон\w*": 0, r"абрахам\w*": 0, r"дрифтвуд": 0, r"мильтон\w*": 0, r"мелвил\w*": 0,
|
||||
r"итан\w*": 0, r"софи": 0, r"рьель": 1, r"фираж\w*": 1, r"грул": 0, r"рукмор\w*": 0,
|
||||
r"фэйрберн\w*": 0, r"майлз\w*": 0, r"дэвис\w*": 0, r"келвин\w*": 0, r"честер\w*": 0,
|
||||
r"сайрус\w*": 0, r"коннор\w*": 0, r"каспер\w*": 0, r"костнер\w*": 0, r"мейнард\w*": 0,
|
||||
r"эксли": 0, r"пауэлл\w*": 0, r"брайтвич\w*": 0, r"димвельд\w*": 0, r"лорэйн\w*": 1,
|
||||
r"ирветт\w*": 1, r"иоланд\w*": 1, r"рэттблик\w*": 0, r"кэйден\w*": 0, r"дэниэл\w*": 0,
|
||||
}
|
||||
_ST = [(re.compile(r"^" + k + r"$", re.I), v) for k, v in STRESS.items()]
|
||||
|
||||
|
||||
def pre(text: str) -> str:
|
||||
for a, b in PRE:
|
||||
text = re.sub(a, b, text)
|
||||
return text
|
||||
|
||||
|
||||
def _mark(word: str, k: int) -> str:
|
||||
out, n = [], 0
|
||||
for ch in word:
|
||||
if ch in VOW:
|
||||
if n == k:
|
||||
out.append("+")
|
||||
n += 1
|
||||
out.append(ch)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def post(plus: str) -> str:
|
||||
"""Fix stresses in RUAccent '+' text: lexicon names, then double marks."""
|
||||
def fix_part(p):
|
||||
clean = p.replace("+", "")
|
||||
for rx, k in _ST:
|
||||
if rx.match(clean):
|
||||
return _mark(clean, k)
|
||||
if p.count("+") > 1:
|
||||
return _mark(clean, sum(1 for ch in p[:p.rindex("+")] if ch in VOW))
|
||||
return p
|
||||
|
||||
def fix_word(m):
|
||||
return "-".join(fix_part(p) for p in m.group(0).split("-"))
|
||||
return re.sub(r"[А-ЯЁа-яё+]+(?:-[А-ЯЁа-яё+]+)*", fix_word, plus)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
for s in ["Кр+а'Т+ау идёт в Пайкм+идз-Рич", "Фир+аж+а и Л+онгк+ип", "Хэрри+от и Конв+эй", "Стилуот+ер-б+эй"]:
|
||||
print(s, "->", post(s))
|
||||
print(pre("Кра'Тау и драк'дал вернулся c разведки «Брэйлин»>, Пыш'Ка"))
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Step 8: build the EML mod into the repository (mod/<id>: the result that is committed).
|
||||
|
||||
usage: mod then `python install.py` copies it into the game and dry-runs it
|
||||
The mod replaces each note's EN sound (the Russian slot does not exist in VoiceoverResource: its fixed
|
||||
languages are EN/DE/FR/ES/ZH): SoundResource.streamedData points to the new QOA content, frameCount and
|
||||
duration follow it, VoiceoverResource.durations[EN] too (10 ms units). The game files are not touched:
|
||||
EML (dbghelp.dll proxy) patches enshrouded.kfc at launch and keeps enshrouded.kfc.bak (emm restore).
|
||||
"""
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
from . import __version__
|
||||
from .config import ROOT, load
|
||||
|
||||
LUA = '''local lines = require("lines")
|
||||
local ok, fail = 0, 0
|
||||
for _, l in ipairs(lines) do
|
||||
local done, err = pcall(function()
|
||||
local snd = game.assets.get_resource(l.sound, "keen::SoundResource", 0)
|
||||
local d = snd.data
|
||||
local content = game.assets.create_content(io.read("data/" .. l.sound .. ".qoa"))
|
||||
d.streamedData.hash = content
|
||||
d.frameCount = l.frames
|
||||
d.framesPerSecond = 48000
|
||||
d.duration.value = math.floor(l.seconds * 1e9)
|
||||
local vo = game.assets.get_resource(l.vo, "keen::VoiceoverResource", 0)
|
||||
vo.data.durations[1] = math.floor(l.seconds * 100)
|
||||
end)
|
||||
if done then ok = ok + 1 else fail = fail + 1; print("[ru_voice] FAIL " .. l.name .. ": " .. tostring(err)) end
|
||||
end
|
||||
print(("[ru_voice] replaced %d notes, %d failed"):format(ok, fail))
|
||||
'''
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
notes = json.load(open(cfg.w("notes.json"), encoding="utf-8"))
|
||||
metas = [json.load(open(cfg.w("qoa", f"{n['k']}.json"))) for n in notes if cfg.w("qoa", f"{n['k']}.json").exists()]
|
||||
if len(metas) < len(notes):
|
||||
print(f"WARNING: {len(metas)} of {len(notes)} notes are assembled; the others stay English")
|
||||
mod = pathlib.Path(cfg.get("mod", "dir") or ROOT / "mod" / cfg.get("mod", "id", "ru_voice_notes"))
|
||||
if mod.exists():
|
||||
shutil.rmtree(mod)
|
||||
(mod / "src").mkdir(parents=True)
|
||||
(mod / "data").mkdir()
|
||||
for m in metas:
|
||||
shutil.copy(cfg.w("qoa", f"{m['sound']}.qoa"), mod / "data" / f"{m['sound']}.qoa")
|
||||
lines = ["-- generated by ensvoice: one entry per voiced note", "return {"]
|
||||
lines += [f' {{ sound = "{m["sound"]}", vo = "{m["vo"]}", frames = {m["frames"]}, seconds = {m["seconds"]:.6f}, '
|
||||
f'name = "{m["name"]}" }},' for m in metas]
|
||||
(mod / "src" / "lines.lua").write_text("\n".join(lines + ["}"]) + "\n", encoding="utf-8")
|
||||
(mod / "src" / "mod.lua").write_text(LUA, encoding="utf-8")
|
||||
json.dump(dict(id=cfg.get("mod", "id", "ru_voice_notes"), name=cfg.get("mod", "name", "RU voice: notes"),
|
||||
version=cfg.get("mod", "version", __version__), capabilities=["patch", "export"],
|
||||
description="Russian narration for the voiced notes (Qwen3-TTS ru-stress, narrator clone, "
|
||||
"automatic QC). Replaces the English voiceover slot.",
|
||||
authors=cfg.get("mod", "authors", [])), open(mod / "mod.json", "w"), indent=2)
|
||||
mb = sum(f.stat().st_size for f in (mod / "data").iterdir()) / 2 ** 20
|
||||
print("mod", mod, len(metas), "notes", round(mb, 1), "MB; install: python install.py")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Step 6: pick the best take of every segment (work/pick.json, work/pick_report.json).
|
||||
|
||||
A take passes when it has no error, a pace of 7-19 chars/s and
|
||||
- segments of up to 3 words (headings, names, signatures): CER <= 0.35 (ASR mangles lone names);
|
||||
- longer ones: WER <= 0.12 or a single wrong word.
|
||||
Among passing takes: score = 6*stress + 0.5*MOS + 2*SIM - 3*WER (stress first: a wrong stress is what
|
||||
a listener hears). No passing take: the lowest WER, and the segment is listed as weak — `render --weak`
|
||||
speaks it in voice.extra_seeds, then qc and pick again.
|
||||
"""
|
||||
import collections
|
||||
import glob
|
||||
import json
|
||||
import re
|
||||
|
||||
import jiwer
|
||||
|
||||
from .config import load
|
||||
|
||||
|
||||
def norm(t):
|
||||
return " ".join(re.sub(r"[^\w\s]", " ", t.lower().replace("ё", "е")).split())
|
||||
|
||||
|
||||
def passes(s, r):
|
||||
if "error" in r or not 7 <= r["cps"] <= 19:
|
||||
return False
|
||||
if len(s["text"].split()) <= 3:
|
||||
h = norm(r.get("hyp", ""))
|
||||
return bool(h) and jiwer.cer(norm(s["text"]), h) <= 0.35
|
||||
return r["wer"] <= 0.12 or r["wer"] * len(norm(s["text"]).split()) <= 1.01
|
||||
|
||||
|
||||
def score(r):
|
||||
return 6 * r["stress_ok"] / max(1, r["stress_n"]) + 0.5 * r.get("mos", 3) + 2 * r["sim"] - 3 * r["wer"]
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
segs = json.load(open(cfg.w("segs.json"), encoding="utf-8"))
|
||||
qc = {}
|
||||
for f in sorted(glob.glob(str(cfg.w("qc", "x").parent / "*.json"))):
|
||||
qc.update(json.load(open(f, encoding="utf-8")))
|
||||
by = collections.defaultdict(dict)
|
||||
for stem, r in qc.items():
|
||||
take, seed = stem.rsplit("_s", 1)
|
||||
by[take][int(seed)] = r
|
||||
pick, weak, missing = {}, [], []
|
||||
for s in segs:
|
||||
takes = by.get(s["take"], {})
|
||||
if not takes:
|
||||
missing.append(s["id"]); continue
|
||||
ok = {sd: r for sd, r in takes.items() if passes(s, r)}
|
||||
if ok:
|
||||
pick[s["id"]] = max(ok, key=lambda sd: score(ok[sd]))
|
||||
else:
|
||||
pick[s["id"]] = min(takes, key=lambda sd: takes[sd]["wer"])
|
||||
weak.append(s["id"])
|
||||
json.dump(pick, open(cfg.w("pick.json"), "w"))
|
||||
take_of = {s["id"]: s["take"] for s in segs}
|
||||
P = [by[take_of[i]][sd] for i, sd in pick.items()]
|
||||
rep = dict(segments=len(segs), picked=len(pick), weak=weak, missing=missing,
|
||||
wer=round(sum(r["wer"] for r in P) / max(1, len(P)), 4),
|
||||
mos=round(sum(r.get("mos", 0) for r in P) / max(1, len(P)), 3),
|
||||
sim=round(sum(r["sim"] for r in P) / max(1, len(P)), 3),
|
||||
stress=round(sum(r["stress_ok"] for r in P) / max(1, sum(r["stress_n"] for r in P)), 4),
|
||||
seeds=dict(collections.Counter(pick.values())))
|
||||
json.dump(rep, open(cfg.w("pick_report.json"), "w"), indent=1)
|
||||
print({k: (len(v) if isinstance(v, list) else v) for k, v in rep.items()})
|
||||
if missing:
|
||||
print(f"{len(missing)} segments have no scored take: run render and qc first")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Step 5: score every take automatically (no listening).
|
||||
|
||||
usage: qc [--shard i/n] [--no-mos] [--f0] [--jobs jobs.json --out out.json]
|
||||
Default jobs: every work/takes/<take>_s<seed>.wav of the current segments; results are cached in
|
||||
work/qc/<i>of<n>.json by take name, so re-runs and lexicon changes only score new takes. Run n shards
|
||||
in parallel to use the GPU fully (each process needs ~8-13 GB RAM and ~5 GB VRAM).
|
||||
|
||||
Per take:
|
||||
- wer, hyp — GigaAM-v3 (RNN-T) on <=20 s pieces cut at silences, against the text;
|
||||
- stress_ok / stress_n — meter2 (shipped with the ru-stress model) agrees with the RUAccent marks;
|
||||
- sim — WavLM-SV x-vector cosine to the EN narrator centroid (timbre);
|
||||
- mos — UTMOSv2 naturalness; cps — pace, chars/s; maxgap — longest inner pause; clip — clipped samples;
|
||||
- f0 — pitch spread in semitones (only with --f0: pyin is slow and runs on the CPU).
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import jiwer
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from .config import load
|
||||
|
||||
VOW = "аеёиоуыэюя"
|
||||
|
||||
|
||||
def norm(t):
|
||||
t = t.lower().replace("ё", "е").replace("́", "").replace("+", "")
|
||||
return " ".join(re.sub(r"[^\w\s]", " ", t).split())
|
||||
|
||||
|
||||
def arg(name, default=None):
|
||||
return sys.argv[sys.argv.index(name) + 1] if name in sys.argv else default
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
sh = tuple(int(x) for x in arg("--shard", "0/1").split("/"))
|
||||
use_mos, use_f0 = "--no-mos" not in sys.argv and cfg.get("qc", "mos", True), "--f0" in sys.argv
|
||||
if arg("--jobs"):
|
||||
jobs, out_path = json.load(open(arg("--jobs"), encoding="utf-8")), arg("--out")
|
||||
else:
|
||||
segs = {s["take"]: s for s in json.load(open(cfg.w("segs.json"), encoding="utf-8"))}
|
||||
jobs = []
|
||||
for f in sorted(glob.glob(str(cfg.w("takes", "x").parent / "*_s*.wav"))):
|
||||
stem = os.path.basename(f)[:-4]
|
||||
take = stem.rsplit("_s", 1)[0]
|
||||
if take in segs:
|
||||
jobs.append(dict(id=stem, text=segs[take]["text"], plus=segs[take]["plus"], wav=f))
|
||||
out_path = str(cfg.w("qc", f"{sh[0]}of{sh[1]}.json"))
|
||||
done = {}
|
||||
for f in glob.glob(str(cfg.w("qc", "x").parent / "*.json")) + ([out_path] if os.path.exists(out_path) else []):
|
||||
try:
|
||||
done.update(json.load(open(f, encoding="utf-8")))
|
||||
except Exception:
|
||||
pass
|
||||
jobs = [j for j in jobs[sh[0]::sh[1]] if j["id"] not in done]
|
||||
print("takes to score", len(jobs), flush=True)
|
||||
if not jobs:
|
||||
return
|
||||
res = json.load(open(out_path, encoding="utf-8")) if os.path.exists(out_path) else {}
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
model = cfg.get("tts", "model", "siriusfreak/qwen3-tts-12hz-1.7b-ru-stress-cf") # meter2 ships with it
|
||||
meter = os.path.join(model if os.path.isdir(model) else snapshot_download(model, allow_patterns=["meter/*"]),
|
||||
"meter")
|
||||
os.environ.setdefault("METER2_HEAD", os.path.join(meter, "head.pt"))
|
||||
os.environ.setdefault("METER2_DEVICE", "cuda")
|
||||
sys.path.insert(0, meter)
|
||||
from meter2 import judge, vowel_pos, words_in
|
||||
|
||||
def marks(plus, text):
|
||||
toks = [m.group(0) for m in re.finditer(r"[А-Яа-яЁё+]+(?:-[А-Яа-яЁё+]+)*", plus)]
|
||||
if len(toks) != len(words_in(text)):
|
||||
return {}
|
||||
return {wi: sum(1 for ch in w[:w.index("+")] if ch.lower() in VOW) for wi, w in enumerate(toks)
|
||||
if "+" in w and len(vowel_pos(w.replace("+", ""))) > 1}
|
||||
|
||||
import gigaam
|
||||
asr = gigaam.load_model("v3_rnnt")
|
||||
tmp = os.path.join(tempfile.gettempdir(), f"ensvoice_qc_{os.getpid()}.wav") # one per process
|
||||
|
||||
def transcribe(y, sr):
|
||||
iv = librosa.effects.split(y, top_db=35, frame_length=1024, hop_length=256)
|
||||
pieces, start, cur = [], None, None
|
||||
for a, b in iv:
|
||||
if cur is None:
|
||||
start, cur = a, b
|
||||
elif (b - start) / sr <= 20:
|
||||
cur = b
|
||||
else:
|
||||
pieces.append((start, cur)); start, cur = a, b
|
||||
if cur is not None:
|
||||
pieces.append((start, cur))
|
||||
out = []
|
||||
for a, b in pieces:
|
||||
sf.write(tmp, y[max(0, a - 2400):b + 2400], sr)
|
||||
r = asr.transcribe(tmp)
|
||||
out.append(r if isinstance(r, str) else getattr(r, "text", str(r)))
|
||||
return " ".join(out), iv
|
||||
|
||||
from transformers import AutoFeatureExtractor, WavLMForXVector
|
||||
sv_id = cfg.get("qc", "wavlm", "microsoft/wavlm-base-plus-sv")
|
||||
fe = AutoFeatureExtractor.from_pretrained(sv_id)
|
||||
sv = WavLMForXVector.from_pretrained(sv_id).cuda().eval()
|
||||
cen = torch.tensor(np.load(cfg.w("centroid.npy")))
|
||||
|
||||
def embed(y16):
|
||||
if len(y16) < 32000:
|
||||
y16 = np.pad(y16, (0, 32000 - len(y16)))
|
||||
x = fe(y16[:16000 * 30], sampling_rate=16000, return_tensors="pt")
|
||||
with torch.no_grad():
|
||||
e = sv(**{k: v.cuda() for k, v in x.items()}).embeddings
|
||||
return torch.nn.functional.normalize(e, dim=-1)[0].cpu()
|
||||
|
||||
if use_mos:
|
||||
import utmosv2
|
||||
mos_model = utmosv2.create_model(pretrained=True)
|
||||
|
||||
for i, j in enumerate(jobs):
|
||||
try:
|
||||
y, sr = sf.read(j["wav"], dtype="float32")
|
||||
if y.ndim > 1:
|
||||
y = y.mean(1)
|
||||
dur = len(y) / sr
|
||||
hyp, iv = transcribe(y, sr)
|
||||
wer = jiwer.wer(norm(j["text"]), norm(hyp)) if norm(hyp) else 1.0
|
||||
J = judge(y, sr, j["text"])
|
||||
items = [(wi, v) for wi, v in marks(j["plus"], j["text"]).items() if wi < len(J) and J[wi]["best"] is not None]
|
||||
y16 = librosa.resample(y, orig_sr=sr, target_sr=16000)
|
||||
gaps = [(iv[k + 1][0] - iv[k][1]) / sr for k in range(len(iv) - 1)]
|
||||
r = dict(dur=round(dur, 2), cps=round(len(j["text"]) / dur, 1), wer=round(wer, 3), hyp=hyp,
|
||||
stress_ok=sum(1 for wi, v in items if J[wi]["best"] == v), stress_n=len(items),
|
||||
bad=[J[wi]["word"] for wi, v in items if J[wi]["best"] != v][:10],
|
||||
sim=round(float(embed(y16) @ cen), 3), maxgap=round(max(gaps), 2) if gaps else 0,
|
||||
clip=int(np.sum(np.abs(y) > 0.99)))
|
||||
if use_f0:
|
||||
f0, vf, _ = librosa.pyin(y16, fmin=60, fmax=400, sr=16000, frame_length=1024)
|
||||
f0 = f0[vf]
|
||||
r["f0"] = round(float(np.std(12 * np.log2(f0 / np.median(f0)))) if len(f0) > 10 else 0.0, 2)
|
||||
if use_mos:
|
||||
r["mos"] = round(float(mos_model.predict(input_path=j["wav"])), 3)
|
||||
except Exception as ex: # a broken take simply loses
|
||||
r = dict(error=str(ex)[:200], wer=1.0, cps=0, stress_ok=0, stress_n=1, sim=0, mos=0)
|
||||
res[j["id"]] = r
|
||||
print(f"{i + 1}/{len(jobs)}", j["id"], r.get("wer"), f'{r["stress_ok"]}/{r["stress_n"]}', r.get("sim"),
|
||||
r.get("mos"), flush=True)
|
||||
if i % 20 == 0:
|
||||
json.dump(res, open(out_path, "w", encoding="utf-8"), ensure_ascii=False)
|
||||
json.dump(res, open(out_path, "w", encoding="utf-8"), ensure_ascii=False)
|
||||
print("QC_DONE", len(res))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Frameless QOA as Enshrouded stores voiceover (SoundResource format FramelessQOA).
|
||||
|
||||
QOA slices without file or frame headers: one little-endian u64 per 20 samples (4-bit scale factor,
|
||||
20 x 3-bit residuals), mono 48 kHz. The LMS state starts at weights [0, 0, -8192, 16384], history 0,
|
||||
and runs on through the whole stream.
|
||||
"""
|
||||
import struct
|
||||
|
||||
DQ = [0.75, -0.75, 2.5, -2.5, 4.5, -4.5, 7, -7]
|
||||
SF = [round((s + 1) ** 2.75) for s in range(16)]
|
||||
DEQ = [[(1 if sf * d >= 0 else -1) * int(abs(sf * d) + 0.5) for d in DQ] for sf in SF]
|
||||
|
||||
|
||||
def _clamp(v):
|
||||
return -32768 if v < -32768 else 32767 if v > 32767 else v
|
||||
|
||||
|
||||
def decode(data: bytes, nsamples: int) -> list:
|
||||
h = [0, 0, 0, 0]
|
||||
w = [0, 0, -(1 << 13), 1 << 14]
|
||||
out, p = [], 0
|
||||
for _ in range((nsamples + 19) // 20):
|
||||
if p + 8 > len(data):
|
||||
break
|
||||
sl, = struct.unpack_from("<Q", data, p)
|
||||
p += 8
|
||||
dq = DEQ[(sl >> 60) & 15]
|
||||
for j in range(20):
|
||||
rd = dq[(sl >> (57 - 3 * j)) & 7]
|
||||
s = _clamp(((h[0] * w[0] + h[1] * w[1] + h[2] * w[2] + h[3] * w[3]) >> 13) + rd)
|
||||
delta = rd >> 4
|
||||
for k in range(4):
|
||||
w[k] += -delta if h[k] < 0 else delta
|
||||
h[0], h[1], h[2], h[3] = h[1], h[2], h[3], s
|
||||
out.append(s)
|
||||
return out[:nsamples]
|
||||
|
||||
|
||||
def encode(samples: list) -> bytes:
|
||||
"""mono int16 samples -> frameless QOA; per slice the scale factor with the least squared error."""
|
||||
h = [0, 0, 0, 0]
|
||||
w = [0, 0, -(1 << 13), 1 << 14]
|
||||
out = bytearray()
|
||||
pad = list(samples) + [0] * ((-len(samples)) % 20)
|
||||
for si in range(0, len(pad), 20):
|
||||
sl, best = pad[si:si + 20], None
|
||||
for sf in range(16):
|
||||
dq, hh, ww, err, bits = DEQ[sf], h[:], w[:], 0, sf
|
||||
for x in sl:
|
||||
pred = (hh[0] * ww[0] + hh[1] * ww[1] + hh[2] * ww[2] + hh[3] * ww[3]) >> 13
|
||||
res = x - pred
|
||||
q = min(range(8), key=lambda k: abs(res - dq[k]))
|
||||
rd = dq[q]
|
||||
s = _clamp(pred + rd)
|
||||
err += (x - s) ** 2
|
||||
if best and err >= best[0]:
|
||||
break
|
||||
de = rd >> 4
|
||||
ww = [a - de if b < 0 else a + de for a, b in zip(ww, hh)]
|
||||
hh = [hh[1], hh[2], hh[3], s]
|
||||
bits = (bits << 3) | q
|
||||
else:
|
||||
if best is None or err < best[0]:
|
||||
best = (err, bits, hh, ww)
|
||||
_, bits, h, w = best
|
||||
out += struct.pack("<Q", bits)
|
||||
return bytes(out)
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Choose the reference clip by numbers: 16 note paragraphs x 2 seeds in every candidate voice.
|
||||
|
||||
usage: refbench [ref_id ...] render (tts env); no ids: the configured reference + the 8 best by sim
|
||||
refbench --report table after `qc --jobs work/refbench/jobs.json --out work/refbench/qc.json`
|
||||
Pick the reference with low WER and few bad takes (a leaking reference shows as many takes with WER > 0.1:
|
||||
English at the start), then the highest MOS; put its id into voice.reference. Local backend only.
|
||||
"""
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import statistics as st
|
||||
import sys
|
||||
|
||||
from .config import load
|
||||
|
||||
|
||||
def report(cfg):
|
||||
q = json.load(open(cfg.w("refbench", "qc.json"), encoding="utf-8"))
|
||||
by = collections.defaultdict(list)
|
||||
for k, r in q.items():
|
||||
if "error" not in r:
|
||||
by[k.split("|")[0]].append(r)
|
||||
rows = []
|
||||
for v, rs in by.items():
|
||||
so = sum(r["stress_ok"] for r in rs) / max(1, sum(r["stress_n"] for r in rs))
|
||||
rows.append((st.mean(r.get("mos", 0) for r in rs), v, len(rs), st.mean(r["wer"] for r in rs),
|
||||
sum(r["wer"] > .1 for r in rs), so, st.mean(r["sim"] for r in rs)))
|
||||
print(f"{'MOS':>5} {'WER':>6} {'bad':>4} {'stress':>7} {'SIM':>6} reference")
|
||||
for mos, v, n, wer, bad, so, sim in sorted(rows, reverse=True):
|
||||
print(f"{mos:5.2f} {wer:6.3f} {bad:4d} {so:7.3f} {sim:6.3f} {v}")
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
if "--report" in sys.argv:
|
||||
return report(cfg)
|
||||
from .accent import acute_marks, plus_marks
|
||||
from .lexicon import post, pre
|
||||
from .render import reference
|
||||
from .tts.local import LocalTTS
|
||||
refs = json.load(open(cfg.w("refs.json"), encoding="utf-8"))
|
||||
ids = [a for a in sys.argv[1:] if not a.startswith("-")] or \
|
||||
list(dict.fromkeys([cfg["voice"]["reference"]] + [c["id"] for c in refs[:8]]))
|
||||
notes = json.load(open(cfg.w("notes.json"), encoding="utf-8"))
|
||||
tp = cfg.w("refbench", "texts.json")
|
||||
if tp.exists():
|
||||
sel = json.load(open(tp, encoding="utf-8"))
|
||||
else:
|
||||
random.seed(7)
|
||||
paras = [(n["k"], p.strip()) for n in notes for p in re.split(r"\n\s*\n", pre(n["text"])) if 150 <= len(p.strip()) <= 380]
|
||||
random.shuffle(paras)
|
||||
seen, sel = set(), []
|
||||
for k, p in paras:
|
||||
if k not in seen and len(sel) < 16:
|
||||
seen.add(k); sel.append(dict(id=k, text=p, plus=post(plus_marks(p))))
|
||||
json.dump(sel, open(tp, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||||
tts, jobs = None, []
|
||||
for rid in ids:
|
||||
wav, text = reference(cfg, rid)
|
||||
tts = tts or LocalTTS(cfg, wav, text)
|
||||
tts.set_voice(wav, text)
|
||||
for seed in (1000, 1001):
|
||||
items = [(acute_marks(s["plus"]), str(cfg.w("refbench", "takes", f"{rid}~{s['id']}~{seed}.wav")))
|
||||
for s in sel]
|
||||
todo = [it for it in items if not os.path.exists(it[1])]
|
||||
if todo:
|
||||
tts.speak(todo, seed)
|
||||
jobs += [dict(id=f"{rid}|{s['id']}|{seed}", text=s["text"], plus=s["plus"], wav=it[1]) for s, it in zip(sel, items)]
|
||||
json.dump(jobs, open(cfg.w("refbench", "jobs.json"), "w", encoding="utf-8"), ensure_ascii=False)
|
||||
print("REFBENCH_RENDERED", len(jobs), "takes; next: python run.py qc --jobs", cfg.w("refbench", "jobs.json"),
|
||||
"--out", cfg.w("refbench", "qc.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Step 2: reference clips of the EN narrator for the voice clone, and the narrator's measurements.
|
||||
|
||||
Candidates: whole notes of 7-16 s and first paragraphs of 7-15 s cut at the pause nearest to where the
|
||||
paragraph should end (by its share of characters). Each needs its exact EN text (ICL cloning), so a cut
|
||||
that misses the pause leaks: the clone then starts Russian takes with the English tail ("we were a
|
||||
harvest...") — refbench catches that by WER.
|
||||
Writes work/refs/<id>.wav (id = "<EN sound name>__all|__p1"), work/refs.json (text, length, similarity
|
||||
to the narrator centroid, F0 spread), work/centroid.npy (mean WavLM x-vector of the narrator, the QC
|
||||
target for the timbre) and work/en_pauses.json (pauses between paragraphs and phrases in the EN originals).
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from .config import load
|
||||
|
||||
|
||||
def clean(s):
|
||||
return re.sub(r"<[^>]+>", "", s or "")
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
notes = json.load(open(cfg.w("notes.json"), encoding="utf-8"))
|
||||
from transformers import AutoFeatureExtractor, WavLMForXVector
|
||||
sv_id = cfg.get("qc", "wavlm", "microsoft/wavlm-base-plus-sv")
|
||||
fe = AutoFeatureExtractor.from_pretrained(sv_id)
|
||||
sv = WavLMForXVector.from_pretrained(sv_id).cuda().eval()
|
||||
|
||||
def embed(y24):
|
||||
y = librosa.resample(y24, orig_sr=24000, target_sr=16000)[:16000 * 30]
|
||||
x = fe(y, sampling_rate=16000, return_tensors="pt")
|
||||
with torch.no_grad():
|
||||
e = sv(**{k: v.cuda() for k, v in x.items()}).embeddings
|
||||
return torch.nn.functional.normalize(e, dim=-1)[0].cpu().numpy()
|
||||
|
||||
cands, embs, para_gaps, sent_gaps = [], [], [], []
|
||||
for n in notes:
|
||||
y, sr = sf.read(cfg.w("en24", f"{n['k']}.wav"), dtype="float32")
|
||||
embs.append(embed(y[: sr * 20]))
|
||||
iv = librosa.effects.split(y, top_db=40, frame_length=1024, hop_length=240)
|
||||
if len(iv) < 2:
|
||||
continue
|
||||
gaps = [((iv[i][1] + iv[i + 1][0]) / 2 / sr, (iv[i + 1][0] - iv[i][1]) / sr) for i in range(len(iv) - 1)]
|
||||
a0, a1 = iv[0][0] / sr, iv[-1][1] / sr
|
||||
paras = [clean(p).strip() for p in re.split(r"\n\s*\n", n["en"] or "") if clean(p).strip()]
|
||||
full = " ".join(paras)
|
||||
if not full:
|
||||
continue
|
||||
if 7 <= a1 - a0 <= 16:
|
||||
cands.append(dict(id=f"{n['name']}__all", k=n["k"], a=a0, b=a1, text=re.sub(r"\s+", " ", full)))
|
||||
if len(paras) >= 2:
|
||||
pos = a0 + (a1 - a0) * len(paras[0]) / len(full)
|
||||
g = min(gaps, key=lambda g: abs(g[0] - pos))
|
||||
if abs(g[0] - pos) < 0.12 * (a1 - a0) and g[1] > 0.35:
|
||||
para_gaps.append(g[1])
|
||||
if 7 <= g[0] - a0 <= 15:
|
||||
cands.append(dict(id=f"{n['name']}__p1", k=n["k"], a=a0, b=g[0] - g[1] / 2 + 0.1,
|
||||
text=re.sub(r"\s+", " ", paras[0])))
|
||||
sent_gaps += [g[1] for g in gaps if g[1] > 0.15]
|
||||
cen = np.mean(embs, 0)
|
||||
cen /= np.linalg.norm(cen)
|
||||
np.save(cfg.w("centroid.npy"), cen)
|
||||
out = []
|
||||
for c in cands:
|
||||
y, sr = sf.read(cfg.w("en24", f"{c['k']}.wav"), dtype="float32")
|
||||
seg = y[max(0, int((c["a"] - 0.15) * sr)): int((c["b"] + 0.1) * sr)]
|
||||
sf.write(cfg.w("refs", f"{c['id']}.wav"), seg, sr)
|
||||
f0, vf, _ = librosa.pyin(librosa.resample(seg, orig_sr=24000, target_sr=16000), fmin=60, fmax=400, sr=16000)
|
||||
f0 = f0[vf]
|
||||
c.update(sec=round(len(seg) / sr, 2), sim=round(float(embed(seg) @ cen), 3),
|
||||
f0=round(float(np.std(12 * np.log2(f0 / np.median(f0)))) if len(f0) > 10 else 0.0, 2))
|
||||
out.append(c)
|
||||
out.sort(key=lambda c: -c["sim"])
|
||||
json.dump(out, open(cfg.w("refs.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
||||
pg, sg = np.array(para_gaps), np.array(sent_gaps)
|
||||
pauses = dict(paragraph_median=float(np.median(pg)), gap_median=float(np.median(sg)),
|
||||
gap_p75=float(np.percentile(sg, 75)))
|
||||
json.dump(pauses, open(cfg.w("en_pauses.json"), "w"), indent=1)
|
||||
print("candidates", len(out), "pauses", pauses)
|
||||
want = cfg["voice"]["reference"]
|
||||
print("configured reference", want, "found" if any(c["id"] == want for c in out) else "NOT FOUND")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Step 4: speak every segment in every seed of voice.seeds (work/takes/<take>_s<seed>.wav).
|
||||
|
||||
usage: render [--weak] --weak: only the segments pick listed as weak, in voice.extra_seeds
|
||||
Resumable: existing takes are skipped, so after a lexicon change only the changed segments are spoken.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
from .config import load
|
||||
from .tts import backend
|
||||
|
||||
|
||||
def reference(cfg, ref_id=None):
|
||||
ref_id = ref_id or cfg["voice"]["reference"]
|
||||
refs = {c["id"]: c for c in json.load(open(cfg.w("refs.json"), encoding="utf-8"))}
|
||||
if ref_id not in refs:
|
||||
raise SystemExit(f"reference {ref_id!r} is not in work/refs.json (run refs; see README)")
|
||||
return cfg.w("refs", f"{ref_id}.wav"), refs[ref_id]["text"]
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
segs = json.load(open(cfg.w("segs.json"), encoding="utf-8"))
|
||||
seeds = cfg["voice"]["seeds"]
|
||||
if "--weak" in sys.argv:
|
||||
weak = set(json.load(open(cfg.w("pick_report.json")))["weak"])
|
||||
segs = [s for s in segs if s["id"] in weak]
|
||||
seeds = cfg["voice"].get("extra_seeds", [])
|
||||
tts = None
|
||||
for seed in seeds:
|
||||
todo = [(s["acute"], str(cfg.w("takes", f"{s['take']}_s{seed}.wav"))) for s in segs
|
||||
if not cfg.w("takes", f"{s['take']}_s{seed}.wav").exists()]
|
||||
print(f"seed {seed}: {len(todo)} segments to speak", flush=True)
|
||||
if not todo:
|
||||
continue
|
||||
if tts is None:
|
||||
tts = backend(cfg, *reference(cfg))
|
||||
tts.speak(todo, seed, log=lambda m: print(m, flush=True))
|
||||
print("RENDER_DONE")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
import re
|
||||
U=["ноль","один","два","три","четыре","пять","шесть","семь","восемь","девять","десять","одиннадцать","двенадцать","тринадцать","четырнадцать","пятнадцать","шестнадцать","семнадцать","восемнадцать","девятнадцать"]
|
||||
T=["","","двадцать","тридцать","сорок","пятьдесят","шестьдесят","семьдесят","восемьдесят","девяносто"]
|
||||
H=["","сто","двести","триста","четыреста","пятьсот","шестьсот","семьсот","восемьсот","девятьсот"]
|
||||
def card(n):
|
||||
if n<20: return U[n]
|
||||
if n<100: return (T[n//10]+(" "+U[n%10] if n%10 else ""))
|
||||
if n<1000: return (H[n//100]+(" "+card(n%100) if n%100 else ""))
|
||||
if n<1000000:
|
||||
th,r=divmod(n,1000); t=th%10; tt=th%100
|
||||
w=card(th) if th>2 or th>=20 else ""
|
||||
if th==1: w="одна тысяча"
|
||||
elif th==2: w="две тысячи"
|
||||
else:
|
||||
w=card(th)
|
||||
w=re.sub(r"один$","одна",w); w=re.sub(r"два$","две",w)
|
||||
w+=" "+("тысяча" if t==1 and tt!=11 else "тысячи" if 2<=t<=4 and not 12<=tt<=14 else "тысяч")
|
||||
return w+(" "+card(r) if r else "")
|
||||
return str(n)
|
||||
ORD={"ноль":"нулев","один":"перв","два":"втор","три":"трет","четыре":"четвёрт","пять":"пят","шесть":"шест","семь":"седьм","восемь":"восьм","девять":"девят","десять":"десят","одиннадцать":"одиннадцат","двенадцать":"двенадцат","тринадцать":"тринадцат","четырнадцать":"четырнадцат","пятнадцать":"пятнадцат","шестнадцать":"шестнадцат","семнадцать":"семнадцат","восемнадцать":"восемнадцат","девятнадцать":"девятнадцат","двадцать":"двадцат","тридцать":"тридцат","сорок":"сороков","пятьдесят":"пятидесят","шестьдесят":"шестидесят","семьдесят":"семидесят","восемьдесят":"восьмидесят","девяносто":"девяност","сто":"сот","двести":"двухсот","триста":"трёхсот","четыреста":"четырёхсот","пятьсот":"пятисот","шестьсот":"шестисот","семьсот":"семисот","восемьсот":"восьмисот","девятьсот":"девятисот","тысяча":"тысячн","тысячи":"тысячн","тысяч":"тысячн"}
|
||||
# ending by case/gender: suffix of written form -> ending (hard stem / трет soft)
|
||||
END={"й":"ый","го":"ого","му":"ому","м":"ом","е":"ое","я":"ая","ю":"ую","х":"ых","ми":"ыми"}
|
||||
def ordinal(n,suf):
|
||||
ws=card(n).split(); last=ws[-1]; stem=ORD.get(last,last)
|
||||
e=END.get(suf,"ый")
|
||||
if stem=="трет": e={"ый":"ий","ого":"ьего","ому":"ьему","ом":"ьем","ое":"ье","ая":"ья","ую":"ью","ых":"ьих","ыми":"ьими"}[e]
|
||||
elif stem in("втор","сороков") or stem.endswith(("тысячн",)):
|
||||
e=e.replace("ый","ой") if stem=="втор" or stem=="сороков" else e
|
||||
if e=="ый" and stem in("втор","шест","седьм","восьм","сороков"): e="ой"
|
||||
return " ".join(ws[:-1]+[stem+e])
|
||||
def normalize(t):
|
||||
t=t.replace("\xa0"," ").replace("•","").replace("*","").replace("_"," ").replace("–",",").replace("№","номер ")
|
||||
t=re.sub(r"(\d+)%",lambda m:card(int(m[1]))+" процентов",t)
|
||||
t=re.sub(r"\b(\d+)-(й|го|му|м|е|я|ю|х|ми)\b",lambda m:ordinal(int(m[1]),m[2]),t)
|
||||
t=re.sub(r"\b(\d+)\s+(году|года|год|годом)\b",lambda m:ordinal(int(m[1]),{"году":"м","года":"го","год":"й","годом":"м"}[m[2]])+" "+m[2],t)
|
||||
t=re.sub(r"^\s*(\d+)\.\s",lambda m:card(int(m[1]))+". ",t,flags=re.M)
|
||||
t=re.sub(r"\d+",lambda m:card(int(m[0])),t)
|
||||
t=re.sub(r"[ \t]{2,}"," ",t)
|
||||
return t
|
||||
if __name__=="__main__":
|
||||
for s in ["в 508 году","3-й ярус","341-го дня","34% урожая","1200 монет","1. Первое","до 507 года","2-ю неделю","3-е место","21 кролик"]: print(s,"->",normalize(s))
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Step 3: the text for the voice, cut into segments (work/segs.json).
|
||||
|
||||
Every note: lexicon.pre -> lines (a blank line = paragraph 'p', a single newline = 'l') -> lines over
|
||||
MAXC characters cut at sentence ends into near-equal pieces ('s') -> RUAccent '+' -> lexicon.post ->
|
||||
U+0301 text for the ru-stress model.
|
||||
A segment's takes are named <id>_<hash>_s<seed>.wav, the hash of the reference and the spoken text:
|
||||
change the lexicon or the reference and only the segments it touches are rendered again.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
|
||||
from .accent import acute_marks, plus_marks
|
||||
from .config import load
|
||||
from .lexicon import post, pre
|
||||
|
||||
MAXC = 380
|
||||
|
||||
|
||||
def split_long(s):
|
||||
if len(s) <= MAXC:
|
||||
return [s]
|
||||
sents = [x.strip() for x in re.findall(r".+?(?:[.!?…]+[»\"')]*(?=\s)|$)\s*", s) if x.strip()]
|
||||
k = math.ceil(len(s) / 330)
|
||||
target, out, cur = len(s) / k, [], ""
|
||||
for x in sents:
|
||||
if cur and len(cur) + len(x) + 1 > target * 1.15 and len(out) < k - 1:
|
||||
out.append(cur); cur = x
|
||||
else:
|
||||
cur = (cur + " " + x).strip()
|
||||
out.append(cur)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load()
|
||||
ref = cfg["voice"]["reference"]
|
||||
notes = json.load(open(cfg.w("notes.json"), encoding="utf-8"))
|
||||
if cfg.only: # ENSVOICE_NOTES=n016,n238: a quick experiment on a few notes
|
||||
notes = [n for n in notes if n["k"] in cfg.only]
|
||||
segs = []
|
||||
for n in notes:
|
||||
units, join = [], "p"
|
||||
for line in pre(n["text"]).split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
join = "p"; continue
|
||||
for j, piece in enumerate(split_long(line)):
|
||||
units.append(("s" if j else join, piece))
|
||||
join = "l"
|
||||
for i, (jn, u) in enumerate(units):
|
||||
plus = post(plus_marks(u))
|
||||
acute = acute_marks(plus)
|
||||
sid = f"{n['k']}_{i:02d}"
|
||||
h = hashlib.sha1(f"{ref}|{acute}".encode()).hexdigest()[:10]
|
||||
segs.append(dict(id=sid, take=f"{sid}_{h}", k=n["k"], i=i, join=jn if i else "-",
|
||||
text=u, plus=plus, acute=acute))
|
||||
json.dump(segs, open(cfg.w("segs.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=0)
|
||||
L = [len(s["text"]) for s in segs]
|
||||
print("segments", len(segs), "notes", len(notes), "longest", max(L), "chars", sum(L))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,16 @@
|
||||
"""TTS backends: speak(items=[(text, out_wav)], seed) in the cloned narrator voice.
|
||||
|
||||
local — qwen-tts in this process on a CUDA GPU (any machine, the default);
|
||||
media_node — an MCP server with generate_speech (the author's home GPU node; see media_node.py).
|
||||
"""
|
||||
|
||||
|
||||
def backend(cfg, ref_wav, ref_text):
|
||||
name = cfg.get("tts", "backend", "local")
|
||||
if name == "local":
|
||||
from .local import LocalTTS
|
||||
return LocalTTS(cfg, ref_wav, ref_text)
|
||||
if name == "media_node":
|
||||
from .media_node import MediaNodeTTS
|
||||
return MediaNodeTTS(cfg)
|
||||
raise SystemExit(f"unknown tts.backend {name!r}")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""qwen-tts (Qwen3-TTS 1.7B, fine-tune siriusfreak/qwen3-tts-12hz-1.7b-ru-stress-cf) on a local GPU.
|
||||
|
||||
ICL voice clone: the reference clip and its exact text are encoded once; texts are spoken in batches
|
||||
of tts.batch with torch.manual_seed(seed + batch start), so a seed is repeatable for the same segments
|
||||
in the same order. fp16 on Volta/Turing, bfloat16 on Ampere and newer (tts.dtype).
|
||||
"""
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
|
||||
class LocalTTS:
|
||||
def __init__(self, cfg, ref_wav, ref_text):
|
||||
from qwen_tts import Qwen3TTSModel
|
||||
t = cfg["tts"]
|
||||
self.model = Qwen3TTSModel.from_pretrained(t.get("model", "siriusfreak/qwen3-tts-12hz-1.7b-ru-stress-cf"),
|
||||
device_map=t.get("device", "cuda:0"),
|
||||
dtype=getattr(torch, t.get("dtype", "bfloat16")))
|
||||
self.batch = int(t.get("batch", 8))
|
||||
self.set_voice(ref_wav, ref_text)
|
||||
|
||||
def set_voice(self, ref_wav, ref_text):
|
||||
self.prompt = self.model.create_voice_clone_prompt(ref_audio=str(ref_wav), ref_text=ref_text)
|
||||
|
||||
def speak(self, items, seed, log=print):
|
||||
items = sorted(items, key=lambda x: len(x[0])) # similar lengths in one batch: less padding
|
||||
for i in range(0, len(items), self.batch):
|
||||
b = items[i:i + self.batch]
|
||||
torch.manual_seed(seed + i)
|
||||
wavs, sr = self.model.generate_voice_clone(text=[t for t, _ in b], language=["Russian"] * len(b),
|
||||
voice_clone_prompt=self.prompt * len(b))
|
||||
for (_, out), w in zip(b, wavs):
|
||||
sf.write(out, w, sr)
|
||||
log(f"seed {seed}: {min(i + self.batch, len(items))}/{len(items)}")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""generate_speech over MCP streamable HTTP (media-node, the author's home GPU node with qwentts.cpp).
|
||||
|
||||
Needs a server with the ru-stress model and a saved project voice (tts.media_node.voice, e.g. made with
|
||||
its save_voice tool from the reference clip). Batches of up to 64 texts / 29000 characters per call.
|
||||
"""
|
||||
import itertools
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
_ids = itertools.count(1)
|
||||
|
||||
|
||||
class MCP:
|
||||
def __init__(self, url):
|
||||
self.url, self.sid = url, None
|
||||
self._post({"jsonrpc": "2.0", "id": next(_ids), "method": "initialize",
|
||||
"params": {"protocolVersion": "2025-06-18", "capabilities": {},
|
||||
"clientInfo": {"name": "ensvoice", "version": "0.4"}}})
|
||||
self._post({"jsonrpc": "2.0", "method": "notifications/initialized"}, expect=False)
|
||||
|
||||
def _post(self, msg, expect=True, timeout=7200):
|
||||
h = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
|
||||
if self.sid:
|
||||
h["Mcp-Session-Id"] = self.sid
|
||||
with urllib.request.urlopen(urllib.request.Request(self.url, json.dumps(msg).encode(), h), timeout=timeout) as r:
|
||||
self.sid = r.headers.get("Mcp-Session-Id", self.sid)
|
||||
body = r.read().decode()
|
||||
if not expect:
|
||||
return None
|
||||
if body.lstrip().startswith("{"):
|
||||
return json.loads(body)
|
||||
last = None
|
||||
for line in body.splitlines():
|
||||
if line.startswith("data:"):
|
||||
d = json.loads(line[5:])
|
||||
if d.get("id") == msg.get("id"):
|
||||
last = d
|
||||
return last
|
||||
|
||||
def call(self, name, args):
|
||||
r = self._post({"jsonrpc": "2.0", "id": next(_ids), "method": "tools/call",
|
||||
"params": {"name": name, "arguments": args}})
|
||||
if "error" in r:
|
||||
raise RuntimeError(r["error"])
|
||||
return r["result"]
|
||||
|
||||
|
||||
class MediaNodeTTS:
|
||||
def __init__(self, cfg):
|
||||
c = cfg["tts"]["media_node"]
|
||||
self.url, self.voice = c["url"], c["voice"]
|
||||
self.rewrite = c.get("download_host_rewrite") # ["from", "to"] if file URLs name another host
|
||||
self.m = MCP(self.url)
|
||||
|
||||
def speak(self, items, seed, log=print):
|
||||
batches, cur, c = [], [], 0
|
||||
for it in items:
|
||||
if cur and (len(cur) == 64 or c + len(it[0]) > 29000):
|
||||
batches.append(cur); cur, c = [], 0
|
||||
cur.append(it); c += len(it[0])
|
||||
if cur:
|
||||
batches.append(cur)
|
||||
for bi, b in enumerate(batches):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = self.m.call("generate_speech", {"model": "ru-stress", "voice": self.voice, "language": "Russian",
|
||||
"seed": seed, "texts": [{"text": t} for t, _ in b]})
|
||||
break
|
||||
except Exception as ex: # a backend timeout: once more with a new session
|
||||
log(f"retry {bi}: {ex}"); self.m = MCP(self.url)
|
||||
files = sorted(r["structuredContent"]["files"], key=lambda f: f["item"])
|
||||
for (_, out), f in zip(b, files):
|
||||
url = f["url"] if not self.rewrite else f["url"].replace(*self.rewrite)
|
||||
urllib.request.urlretrieve(url, out)
|
||||
log(f"seed {seed}: batch {bi + 1}/{len(batches)}")
|
||||
Reference in New Issue
Block a user