From a91909f731feb5dd476fffe098b9b8d2b282a7f5 Mon Sep 17 00:00:00 2001 From: Wartana Date: Tue, 3 Mar 2026 22:13:30 +0800 Subject: [PATCH] feat: Implement SSH-triggered BGP route export with log polling and validation, and update queue limits with a new profile. --- bgp_export_script.rsc | 37 ++++++++++++++------- run_bgp_sync.sh | 3 +- sync_routing.py | 73 ++++++++++++++++++++++++++++++++++++++---- update_mangle_queue.py | 19 +++++------ 4 files changed, 104 insertions(+), 28 deletions(-) diff --git a/bgp_export_script.rsc b/bgp_export_script.rsc index 039637b..a4f01a8 100644 --- a/bgp_export_script.rsc +++ b/bgp_export_script.rsc @@ -1,23 +1,36 @@ /system/script remove [find name="bgp_lokal_export"] /system/script add name=bgp_lokal_export dont-require-permissions=yes source={ \ -:log info "BGP Export: mulai..."; \ +:local startTime [/system clock get time]; +:local startDate [/system clock get date]; +:log info "BGP Export [$startDate $startTime]: mulai..."; /ip firewall address-list remove [find list="bgp-export"]; \ :local cnt 0; \ -:foreach r in=[/routing/route find where distance=15] do={ \ - :local dst [/routing/route get $r dst-address]; \ - :if ($dst != "0.0.0.0/0" && $dst != "::/0") do={ \ - /ip firewall address-list add list="bgp-export" address=$dst; \ - :set cnt ($cnt + 1); \ +\ +# Iterasi distance=15 (CDN) - pecah per prefix /8 untuk menghindari limit array ~8192 RouterOS \ +:foreach pref in={1;2;5;10;14;17;23;27;31;36;42;43;45;49;58;59;60;61;62;63;64;65;66;67;68;69;70;71;72;73;74;75;76;77;78;79;80;81;82;83;84;85;86;87;88;89;90;91;92;93;94;95;96;97;98;99;100;101;102;103;104;105;106;107;108;109;110;111;112;113;114;115;116;117;118;119;120;121;122;123;124;125;126;127;128;129;130;131;132;133;134;135;136;137;138;139;140;141;142;143;144;145;146;147;148;149;150;151;152;153;154;155;156;157;158;159;160;161;162;163;164;165;166;167;168;169;170;171;172;173;174;175;176;177;178;179;180;181;182;183;184;185;186;187;188;189;190;191;192;193;194;195;196;197;198;199;200;201;202;203;204;205;206;207;208;209;210;211;212;213;214;215;216;217;218;219;220;221;222;223;224;225;226;227;228;229;230;231;232;233;234;235;236;237;238;239;240;241;242;243;244;245;246;247;248;249;250;251;252;253;254;255} do={ \ + :local subnet ($pref . ".0.0.0/8"); \ + :foreach r in=[/routing/route find where distance=15 dst-address~("^" . $pref . "\\.")] do={ \ + :local dst [/routing/route get $r dst-address]; \ + :if ($dst != "0.0.0.0/0" && $dst != "::/0") do={ \ + /ip firewall address-list add list="bgp-export" address=$dst; \ + :set cnt ($cnt + 1); \ + }; \ }; \ }; \ :log info "BGP Export: CDN selesai ($cnt)"; \ -:foreach r in=[/routing/route find where distance=200] do={ \ - :local dst [/routing/route get $r dst-address]; \ - :if ($dst != "0.0.0.0/0" && $dst != "::/0") do={ \ - /ip firewall address-list add list="bgp-export" address=$dst; \ - :set cnt ($cnt + 1); \ +\ +# Iterasi distance=200 (IIX Lokal) - pecah per prefix /8 \ +:foreach pref in={1;2;5;10;14;17;23;27;31;36;42;43;45;49;58;59;60;61;62;63;64;65;66;67;68;69;70;71;72;73;74;75;76;77;78;79;80;81;82;83;84;85;86;87;88;89;90;91;92;93;94;95;96;97;98;99;100;101;102;103;104;105;106;107;108;109;110;111;112;113;114;115;116;117;118;119;120;121;122;123;124;125;126;127;128;129;130;131;132;133;134;135;136;137;138;139;140;141;142;143;144;145;146;147;148;149;150;151;152;153;154;155;156;157;158;159;160;161;162;163;164;165;166;167;168;169;170;171;172;173;174;175;176;177;178;179;180;181;182;183;184;185;186;187;188;189;190;191;192;193;194;195;196;197;198;199;200;201;202;203;204;205;206;207;208;209;210;211;212;213;214;215;216;217;218;219;220;221;222;223;224;225;226;227;228;229;230;231;232;233;234;235;236;237;238;239;240;241;242;243;244;245;246;247;248;249;250;251;252;253;254;255} do={ \ + :foreach r in=[/routing/route find where distance=200 dst-address~("^" . $pref . "\\.")] do={ \ + :local dst [/routing/route get $r dst-address]; \ + :if ($dst != "0.0.0.0/0" && $dst != "::/0") do={ \ + /ip firewall address-list add list="bgp-export" address=$dst; \ + :set cnt ($cnt + 1); \ + }; \ }; \ }; \ -:log info "BGP Export: selesai total $cnt rute"; \ +:local endTime [/system clock get time]; +:local endDate [/system clock get date]; +:log info "BGP Export [$endDate $endTime]: selesai total $cnt rute"; \ } diff --git a/run_bgp_sync.sh b/run_bgp_sync.sh index 178dffd..593f742 100755 --- a/run_bgp_sync.sh +++ b/run_bgp_sync.sh @@ -1,6 +1,6 @@ #!/bin/bash # AUTO-SYNC BGP ROUTING TO DISTRIBUTION ROUTERS -# Dijalankan via Cron Job +# Dijalankan via Cron Job jam 04:00 cd /home/wartana/myApp/iix/ source venv/bin/activate @@ -9,6 +9,7 @@ echo "==================================================" echo "Memulai Sinkronisasi BGP: $(date)" # 1. Ekstrak rute BGP Lokal dari Core (menghasilkan routing-lokal.rsc) +# Script ini secara internal akan melakukan call SSH dan log polling python3 sync_routing.py if [ $? -eq 0 ]; then diff --git a/sync_routing.py b/sync_routing.py index e44c69c..3cf22b2 100644 --- a/sync_routing.py +++ b/sync_routing.py @@ -20,15 +20,34 @@ auth = (BGP_ROUTER_USER, BGP_ROUTER_PASSWORD) def get_local_bgp_routes(): """Mengambil daftar IP lokal dari address-list 'bgp-export' di router BGP. - Address-list ini diisi secara berkala oleh scheduler MikroTik (bgp_lokal_scheduler) - yang menjalankan script bgp_lokal_export setiap hari jam 04:00. - Script mengumpulkan dst-address dari rute CDN (distance=15) dan NIX (distance=200) - lalu menulisnya ke address-list 'bgp-export'. + Prosedurnya: + 1. Triggers script 'bgp_lokal_export' via SSH background command. + 2. API polling ke /log MikroTik mencari string 'BGP Export: selesai'. + 3. Setelah tercapai secara proporsional (~15K IPs), list diekstrak. """ import urllib3 + import time + import subprocess + from datetime import datetime urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - print(f"Menghubungi router {BGP_ROUTER_IP} via REST API...") + print(f"Menghubungi router {BGP_ROUTER_IP} via SSH untuk trigger script...") + + # 1. Jalankan script di Mikrotik via SSH background command + try: + ssh_cmd = [ + "sshpass", "-p", BGP_ROUTER_PASSWORD, + "ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", + f"{BGP_ROUTER_USER}@{BGP_ROUTER_IP}", + "/system/script run bgp_lokal_export" + ] + # Run process asynchronous di background (-n/& setara dengan Popen) + # Jangan terminate proses Python sebelum SSH selesai, agar RouterOS tidak kill script-nya! + subprocess.Popen(ssh_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + print(" -> Script 'bgp_lokal_export' berhasil di-trigger via SSH.") + except Exception as e: + print(f" -> Gagal trigger script via SSH: {e}") + print(" -> Melanjutkan dengan data bgp-export yang ada (jika ada).") session = requests.Session() session.auth = auth @@ -36,11 +55,41 @@ def get_local_bgp_routes(): api_url = f"http://{BGP_ROUTER_IP}/rest" + # 2. Polling Log MikroTik setiap 5 detik hingga ketemu indikasi BGP selesai + print("Memonitor log MikroTik untuk indikasi selesai...") + found_completion = False + start_time = time.time() + max_wait_time = 600 # 10 menit maksimal wait + + import re + while time.time() - start_time < max_wait_time: + try: + log_res = session.get(f"{api_url}/log", timeout=10) + if log_res.status_code == 200: + logs = log_res.json() + # Batasi pengecekan pada 50 baris terakhir log + recent_logs = logs[-50:] if len(logs) > 50 else logs + for lg in recent_logs: + msg = lg.get("message", "") + if re.search(r"BGP Export.*selesai total", msg): + print(f" -> Indikator selesai terdeteksi di log: '{msg}'") + found_completion = True + break + if found_completion: + break + except Exception as e: + pass + time.sleep(5) + + if not found_completion: + print(" -> Waktu tunggu habis tapi tidak menemukan indikator log 'selesai total'.") + print(" -> Melanjutkan mencoba membaca bgp-export...") + print("Mengambil address-list 'bgp-export' dari router...") try: res = session.get(f"{api_url}/ip/firewall/address-list", params={"list": "bgp-export", ".proplist": "address"}, - timeout=30) + timeout=60) if res.status_code == 200: data = res.json() @@ -87,8 +136,20 @@ if __name__ == "__main__": print(f"\nRingkasan:") print(f"Total Subnet IP Lokal & CDN: {len(local_ips)}") + # SAFETY CROSS-CHECK: Pastikan jumlah rute masuk akal + # Karena rata-rata BGP lokal ada ~15.000+, kita set batas minimal aman 13.000 + if len(local_ips) < 13000: + print(f"\n[ERROR] Validasi GAGAL! Rute yang diekstrak hanya {len(local_ips)}.") + print("Terdapat kemungkinan proses export di Router BGP belum selesai atau terputus.") + print("Membatalkan seluruh deployment untuk mencegah router distribusi kehilangan address list.") + exit(1) + + print(f"Membuat file routing-lokal.rsc dengan {len(local_ips)} subnet IP...") # Generate script address-list khusus untuk trafik Lokal generate_address_list_script(local_ips, "ip-lokal", "routing-lokal.rsc") print("\nSelesai! File routing-lokal.rsc siap di-upload ke router distribusi.") print("Di Router Distribusi jalankan perintah: /import file-name=routing-lokal.rsc") + else: + print("\nGagal mengambil rute BGP lokal. File routing-lokal.rsc tidak dibuat.") + exit(1) diff --git a/update_mangle_queue.py b/update_mangle_queue.py index d069e29..d7e0030 100644 --- a/update_mangle_queue.py +++ b/update_mangle_queue.py @@ -62,6 +62,7 @@ def update_mangle_and_queue(): ("star_20", 20, 10, "3G", "500M"), ("star_30", 30, 15, "3G", "500M"), ("star_50", 50, 25, "3G", "500M"), + ("promo_50", 50, 10, "3G", "500M"), ("star_100", 100, 50, "3G", "500M"), ("star_150", 150, 75, "3G", "500M"), ("star_200", 200, 100, "3G", "500M"), @@ -73,8 +74,8 @@ def update_mangle_and_queue(): # --- A. SETUP QUEUE TYPES & QUEUE PARENT LOKAL --- # Bikin Queue Parent LOKAL - commands.append("/queue/tree/add max-limit=5G name=1_all_dl_Dimensi_LOKAL parent=global queue=default") - commands.append("/queue/tree/add max-limit=5G name=5_all_ul_Dimensi_LOKAL parent=global queue=default") + commands.append("/queue/tree/add max-limit=3G name=1_all_dl_Dimensi_LOKAL parent=global queue=default") + commands.append("/queue/tree/add max-limit=3G name=5_all_ul_Dimensi_LOKAL parent=global queue=default") print("Menyusun skrip untuk Profil Spesial (EXPIRED & Hemat)...") @@ -94,8 +95,8 @@ def update_mangle_and_queue(): commands.append("/ip/firewall/mangle/add action=mark-packet chain=forward dst-address-list=hemat new-packet-mark=hemat_dl passthrough=no") commands.append("/ip/firewall/mangle/add action=mark-packet chain=forward src-address-list=hemat new-packet-mark=hemat_ul passthrough=no") # Child Queue Lokal HEMAT - commands.append("/queue/tree/add max-limit=5G name=hemat_dl_local packet-mark=hemat_dl_local parent=1_all_dl_Dimensi_LOKAL queue=hemat_dl_6m") - commands.append("/queue/tree/add max-limit=5G name=hemat_ul_local packet-mark=hemat_ul_local parent=5_all_ul_Dimensi_LOKAL queue=hemat_ul_6m") + commands.append("/queue/tree/add max-limit=3G name=hemat_dl_local packet-mark=hemat_dl_local parent=1_all_dl_Dimensi_LOKAL queue=hemat_dl_6m") + commands.append("/queue/tree/add max-limit=3G name=hemat_ul_local packet-mark=hemat_ul_local parent=5_all_ul_Dimensi_LOKAL queue=hemat_ul_6m") # Update Queue Int Lama untuk sinkronisasi pembaruan nama packet-mark `_ul` commands.append("/queue/tree/set [find name=\"hemat_up\"] name=\"hemat_ul\" packet-mark=hemat_ul queue=hemat_ul_6m") @@ -129,9 +130,9 @@ def update_mangle_and_queue(): commands.append(f"/ip/firewall/mangle/add action=mark-packet chain=forward src-address-list={name} new-packet-mark={up_int} passthrough=no") # 2. UBAH/BUAT BIKIN QUEUE TYPES SETENGAH (Hanya jika belum ada). - # Kita menggunakan _half sebagai penanda - pcq_dl_half = f"pcq_dl_{full}m_half" - pcq_up_half = f"pcq_ul_{full}m_half" + # Kita menggunakan _int sebagai penanda antrean limitasi beda rasio + pcq_dl_half = f"pcq_dl_{half}m_int" + pcq_up_half = f"pcq_ul_{half}m_int" commands.append(f"/queue/type/add kind=pcq name={pcq_dl_half} pcq-classifier=dst-address pcq-rate={half}M") commands.append(f"/queue/type/add kind=pcq name={pcq_up_half} pcq-classifier=src-address pcq-rate={half}M") @@ -152,8 +153,8 @@ def update_mangle_and_queue(): commands.append(f"/queue/tree/set [find name=\"{name}_up\"] name=\"{name}_ul\" packet-mark={up_int} queue={pcq_up_half}") # 4. BIKIN CHILD QUEUE LOKAL YANG BARU (Gunakan rate FULL / Queue eksisting PCQ ori) - commands.append(f"/queue/tree/add max-limit=5G name={name}_dl_local packet-mark={dl_loc} parent=1_all_dl_Dimensi_LOKAL queue={pcq_dl_full}") - commands.append(f"/queue/tree/add max-limit=5G name={name}_ul_local packet-mark={up_loc} parent=5_all_ul_Dimensi_LOKAL queue={q_up_ori}") + commands.append(f"/queue/tree/add max-limit=3G name={name}_dl_local packet-mark={dl_loc} parent=1_all_dl_Dimensi_LOKAL queue={pcq_dl_full}") + commands.append(f"/queue/tree/add max-limit=3G name={name}_ul_local packet-mark={up_loc} parent=5_all_ul_Dimensi_LOKAL queue={q_up_ori}") # --- EKSEKUSI SEMUA COMMAND BARIS DEMI BARIS --- print(f"Mengeksekusi {len(commands)} baris konfigurasi secara antrean (Sequential)...")