Squashed commit of the following:

commit 9a5e1af9b969e3cbacdab1ece7ef25190194b3d5 Author: Joppe Blondel <joppe@blondel.nl> Date: Sun Sep 4 19:32:35 2022 +0200 Cleaned up tree Signed-off-by: Joppe Blondel <joppe@blondel.nl> commit 5f5556409a71afc904bb9df0915cd236d87fccb1 Author: Joppe Blondel <joppe@blondel.nl> Date: Sun Sep 4 19:31:20 2022 +0200 Split up different scripts Signed-off-by: Joppe Blondel <joppe@blondel.nl> commit 6855e9a1e808a99c4a326be7ef49b9b545eaf4bd Author: Jojojoppe <joppe@blondel.nl> Date: Sun Sep 4 14:21:35 2022 +0200 Client server structure done Signed-off-by: Jojojoppe <joppe@blondel.nl> commit 44923b8b3407adb1f8f1c0d24c016613da68a726 Author: Jojojoppe <joppe@blondel.nl> Date: Sat Sep 3 22:35:00 2022 +0200 Moved basic stuff to exec_class Signed-off-by: Jojojoppe <joppe@blondel.nl> Signed-off-by: Joppe Blondel <joppe@blondel.nl>
2022-09-04 19:34:07 +02:00
parent 162aaf47a0
commit 45eb980984
20 changed files with 698 additions and 231 deletions
--- a/scripts/rbuild
+++ b/scripts/rbuild
@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import configparser
+import sys
+
+def print_help():
+    print("Unified FPGA synthesizer frontend\r\n(c) Joppe Blondel - 2022\r\n")
+    print(f"Usage: {sys.argv[0]} [ OPTIONS ] action [ target ] ...")
+    print("")
+    print("Options:")
+    print("  -h                 Show this help message")
+    print("  -c <file>          Configuration file, defaults to project.cfg")
+    print("")
+    print("Actions:")
+    print("ip <target>          Generate IP files from vendor provided libraries")
+    print("syn <target>         Synthesize design for target")
+    print("impl <target>        Route and place design for target")
+    print("bit <target>         Generate output files and run analysis for target")
+    print("all <target>         Generate IP, synthesize, route and place design for target")
+    print("floorplan <target>   Run floorplan editor, currently only for local execution")
+    print("sim <simtarget>      Run simulation target")
+
+if __name__=="__main__":
+    # Parse arguments
+    i = 1
+    nextarg = None
+    configpath = 'project.cfg'
+    actions = []
+    while i<len(sys.argv):
+        if nextarg is not None:
+            if nextarg=='config':
+                configpath = sys.argv[i]
+                nextarg = None
+            else:
+                actions.append((nextarg, sys.argv[i]))
+                nextarg = None
+        elif sys.argv[i]=='-h':
+            print_help()
+            exit(0)
+        elif sys.argv[i]=='-c':
+            nextarg = 'config'
+        else:
+            nextarg = sys.argv[i]
+        i += 1
+    if nextarg is not None:
+        print("ERROR: expected more arguments")
+        exit(1)
+
+    config = configparser.ConfigParser()
+    config.read(configpath)
+
+    subprocesses = []
+
+    try:
+        for action in actions:
+            target = action[1]
+            action = action[0]
+
+            if not config.has_section(f'build:{target}'):
+                print("ERROR: config file has no build section for target")
+                exit(1)
+            devtarget = f'target:{config.get(f"build:{target}", "target", fallback="")}'
+            if not config.has_section(devtarget):
+                print("ERROR: config file has no section for device target")
+                exit(1)
+            toolchain = config.get(devtarget, 'toolchain', fallback="NONE")
+            if toolchain=="NONE":
+                print("ERROR: no toolchain specified for device target")
+                exit(1)
+            
+            try:
+                exec(f"from remotesyn.{toolchain}.{action} import do")
+            except ImportError:
+                print(f"ERROR: Unknown action '{action}' for toolchain '{toolchain}'")
+                exit(1)
+
+            ret = do(config, target, print, subprocesses)
+
+            if ret!=0:
+                exit(ret)
+
+    except KeyboardInterrupt:
+        print("\rStopping rbuild")
+        for p in subprocesses:
+            p.kill()
+        exit(0)
--- a/scripts/rmbuild
+++ b/scripts/rmbuild
@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+import configparser
+import sys
+import paramiko
+import base64
+import struct
+import os
+import json
+
+def cmd(cmd, channel):
+    channel.exec_command(base64.encodebytes(cmd))
+
+def sstr(s):
+    return struct.pack('>I', len(s)) + s.encode('utf-8')
+    
+def rstr(channel):
+    l = struct.unpack('>I', channel.recv(4))[0]
+    return bytes.decode(channel.recv(l), 'utf-8')
+
+def send_file(channel, file, othername=None):
+    print(f"> {file}")
+    if not os.path.exists(file):
+        print(f"Error: {file} does not exists")
+    with open(file, 'rb') as f:
+        stat = os.fstat(f.fileno())
+        print('  -> fsize', stat.st_size)
+        if othername is None:
+            othername = file
+        fsize = struct.pack('>q', stat.st_size)
+        cmd(b'sf'+sstr(othername)+fsize, channel)
+        status = channel.recv(3)
+        if status!=b'OK\n':
+            print('Something went wrong...')
+            exit(1)
+        i = stat.st_size
+        while i>0:
+            fdata = f.read(1024)
+            i -= 1024
+            channel.sendall(fdata)
+
+def recv_file(channel, file):
+    print(f"< {file}")
+    if os.path.dirname(file) != '':
+        os.makedirs(os.path.dirname(file), exist_ok=True)
+    with open(file, 'wb') as f:
+        cmd(b'rf'+sstr(file), channel)
+        while True:
+            status = channel.recv(2)
+            if status != b'\x00\x00':
+                break
+        if status!=b'OK':
+            msg = channel.recv(1024)
+            print("Error:", bytes.decode(msg, 'ascii'))
+            exit(1)
+        fsize = channel.recv(8)
+        fsize = struct.unpack('>q', fsize)[0]
+        print('  -> fsize', fsize)
+        while fsize>0:
+            f.write(channel.recv(1024))
+            fsize -= 1024
+
+def print_help():
+    print("Unified FPGA synthesizer frontend - remote execution\r\n(c) Joppe Blondel - 2022\r\n")
+    print(f"Usage: {sys.argv[0]} [ OPTIONS ] action [ target ] ...")
+    print("")
+    print("Options:")
+    print("  -h                 Show this help message")
+    print("  -c <file>          Configuration file, defaults to project.cfg")
+    print("")
+    print("Actions:")
+    print("ip <target>          Generate IP files from vendor provided libraries")
+    print("syn <target>         Synthesize design for target")
+    print("impl <target>        Route and place design for target")
+    print("bit <target>         Generate output files and run analysis for target")
+    print("all <target>         Generate IP, synthesize, route and place design for target")
+    print("floorplan <target>   Run floorplan editor, currently only for local execution")
+    print("sim <simtarget>      Run simulation target")
+
+if __name__=="__main__":
+    # Parse arguments
+    i = 1
+    nextarg = None
+    configpath = 'project.cfg'
+    actions = []
+    while i<len(sys.argv):
+        if nextarg is not None:
+            if nextarg=='config':
+                configpath = sys.argv[i]
+                nextarg = None
+            else:
+                actions.append((nextarg, sys.argv[i]))
+                nextarg = None
+        elif sys.argv[i]=='-h':
+            print_help()
+            exit(0)
+        elif sys.argv[i]=='-c':
+            nextarg = 'config'
+        else:
+            nextarg = sys.argv[i]
+        i += 1
+    if nextarg is not None:
+        print("ERROR: expected more arguments")
+        exit(1)
+
+    config = configparser.ConfigParser()
+    config.read(configpath)
+
+    # Get SSH configuration
+    privkey = config.get('server', 'privkey', fallback='__privkey__')
+    pubkey = config.get('server', 'pubkey', fallback='__pubkey__')
+    hostname = config.get('server', 'hostname', fallback='__hostname__')
+    port = config.get('server', 'port', fallback='__port__')
+    if privkey=='__privkey__' or pubkey=='__pubkey__' or hostname=='__hostname__' or port=='__port__':
+        print("ERROR: Not enough server information in the config file")
+        exit(1)
+
+    # Connect to SSH and create channel
+    try:
+        host_key = paramiko.RSAKey(filename=privkey)
+        client = paramiko.SSHClient()
+        client.load_system_host_keys()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        trans = paramiko.Transport((hostname, int(port)))
+        trans.connect(None, pkey=host_key)
+        channel = trans.open_channel('session')
+    except paramiko.ssh_exception.SSHException as e:
+        print("ERROR: Could not connect to server")
+        exit(1)
+
+    # Send project identification
+    cmd(b'id' + struct.pack('>q', hash(host_key.get_base64())), channel)
+    # Send config
+    cmd(b'cf' + sstr(json.dumps({s:dict(config.items(s)) for s in config.sections()})), channel)
+
+    subprocesses = []
+
+    try:
+        for action in actions:
+            target = action[1]
+            action = action[0]
+
+            if not config.has_section(f'build:{target}'):
+                print("ERROR: config file has no build section for target")
+                exit(1)
+            devtarget = f'target:{config.get(f"build:{target}", "target", fallback="")}'
+            if not config.has_section(devtarget):
+                print("ERROR: config file has no section for device target")
+                exit(1)
+            toolchain = config.get(devtarget, 'toolchain', fallback="NONE")
+            if toolchain=="NONE":
+                print("ERROR: no toolchain specified for device target")
+                exit(1)
+            
+            try:
+                exec(f"from remotesyn.{toolchain}.{action} import do, needed_files, generated_files")
+            except ImportError:
+                print(f"ERROR: Unknown action '{action}' for toolchain '{toolchain}'")
+                exit(1)
+
+            # Send needed files
+            for f in needed_files(config, target):
+                send_file(channel, f)
+
+            # ret = do(config, target, print, subprocesses)
+            cmd(b'do'+sstr(f"{action} {target}"), channel)
+            ret = 0
+
+            # Get generated files
+            for f in generated_files(config, target):
+                recv_file(channel, f)
+
+            if ret!=0:
+                exit(ret)
+
+    except paramiko.ssh_exception.SSHException as e:
+        print("ERROR: Connection error...")
+        for p in subprocesses:
+            p.kill()
+        exit(0)
+
+    except KeyboardInterrupt:
+        print("\rStopping rmbuild")
+        for p in subprocesses:
+            p.kill()
+        exit(0)
--- a/scripts/rmserver
+++ b/scripts/rmserver
@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+
+import configparser
+import signal
+import sys
+import time
+import subprocess
+from types import NoneType
+import paramiko
+import base64
+import struct
+import os
+import json
+import threading
+import socket
+import shutil
+
+# List of running threads
+threads = []
+running = False
+
+def sighandler(sig, frame):
+    global threads
+    global running
+
+    print("\rStopping server")
+    running = False
+    for t in threads:
+        t.stop();
+
+class FileTransferSF(threading.Thread):
+    def __init__(self, channel, fname, identifier, fsize):
+        threading.Thread.__init__(self)
+        self.channel = channel
+        self.fname = fname
+        self.identifier = identifier
+        self.fsize = fsize
+        self.running = True
+    def stop(self):
+        self.running = False
+    def run(self):
+        with open(f"{self.identifier}/{self.fname}", 'wb') as f:
+            fsize = self.fsize
+            while fsize>0 and self.running:
+                fdata = self.channel.recv(1024)
+                f.write(fdata)
+                fsize -= 1024
+
+class FileTransferRF(threading.Thread):
+    def __init__(self, channel, fname, identifier):
+        threading.Thread.__init__(self)
+        self.channel = channel
+        self.fname = fname
+        self.identifier = identifier
+        self.running = True
+    def stop(self):
+        self.running = False
+    def run(self):
+        with open(f"{self.identifier}/{self.fname}", 'rb') as f:
+            stat = os.fstat(f.fileno())
+            print('  -> fsize', stat.st_size)
+            fsize = struct.pack('>q', stat.st_size)
+            i = stat.st_size
+            self.channel.sendall(b'OK'+fsize)
+            while i>0 and self.running:
+                fdata = f.read(1024)
+                self.channel.sendall(fdata)
+                i -= 1024
+
+class SSHServer(paramiko.ServerInterface):
+    def __init__(self, authorized):
+        self.event = threading.Event()
+        self.authorized = authorized
+        self.identifier = ''
+        self.subprocesses = []
+        self.threads = []
+
+    def stop(self):
+        self.event.set()
+        for s in self.subprocesses:
+            s.kill()
+        for t in self.threads:
+            if type(t) is not NoneType:
+                t.stop()
+                t.join()
+
+    def check_channel_request(self, kind, chanid):
+        if kind == 'session':
+            return paramiko.OPEN_SUCCEEDED
+
+    def check_auth_publickey(self, username, key):
+        keyascii = key.get_base64()
+        for auth in self.authorized:
+            authascii = auth.split(' ')[1]
+            if authascii==keyascii:
+                return paramiko.AUTH_SUCCESSFUL
+        return paramiko.AUTH_FAILED
+
+    def get_allowed_auths(self, username):
+        return 'publickey'
+
+    def rstr(self, b):
+        l = struct.unpack('>I', b[:4])[0]
+        return (bytes.decode(b[4:4+l], 'utf-8'), b[4+l:])
+
+    def sstr(self, s):
+        return struct.pack('>I', len(s)) + s.encode('utf-8')
+
+    def check_channel_exec_request(self, channel, command):
+        try:
+            command = base64.decodebytes(command)
+            cmd = command[:2]
+            data = command[2:]
+
+            # Identifier
+            if cmd==b'id':
+                identifier = struct.unpack('>q', data[:8])[0]
+                self.identifier = str(identifier)
+                print('>', identifier)
+                # Create directory
+                if os.path.exists(str(identifier)):
+                    shutil.rmtree(str(identifier))
+                os.mkdir(str(identifier))
+            
+            # Exit
+            elif cmd==b'ex':
+                print('<', self.identifier)
+                self.event.set()
+
+            # Config
+            elif cmd==b'cf':
+                cnf, data = self.rstr(data)
+                self.config = configparser.ConfigParser()
+                self.config.read_dict(json.loads(cnf))
+
+            # List files
+            elif cmd==b'ls':
+                dr, data = self.rstr(data)
+                print('ls', dr)
+                if not os.path.exists(f"{self.identifier}/{dr}"):
+                    channel.sendall(b'ERFile not found')
+                es = []
+                for f in os.listdir(f'{self.identifier}/{dr}'):
+                    if os.path.isfile(f'{self.identifier}/{dr}/{f}'):
+                        df = 'f'
+                    else:
+                        df = 'd'
+                    es.append(f'{df}{f}')
+                channel.sendall(b'OK' + self.sstr('\n'.join(es)))
+
+            # Send file
+            elif cmd==b'sf':
+                fname, data = self.rstr(data)
+                fsize = struct.unpack('>q', data)[0]
+                print('>>', fname, fsize)
+                os.makedirs(os.path.dirname(f"{self.identifier}/{fname}"), exist_ok=True)
+                channel.sendall(b'OK\n')
+                t = FileTransferSF(channel, fname, self.identifier, fsize)
+                self.threads.append(t)
+                t.start()
+
+            # Receive file
+            elif cmd==b'rf':
+                fname, data = self.rstr(data)
+                print('<<', fname, self.identifier)
+                if not os.path.exists(f"{self.identifier}/{fname}"):
+                    channel.sendall(b'ERFile not found')
+                else:
+                    t = FileTransferRF(channel, fname, self.identifier)
+                    self.threads.append(t)
+                    t.start()
+
+            # Execute rbuild
+            elif cmd==b'do':
+                args, data = self.rstr(data)
+                print('[]', args)
+                with open(f"{self.identifier}/project.cfg", "w") as f:
+                    self.config.write(f)
+                p = subprocess.Popen(f"rbuild -c project.cfg {args}", shell=True, cwd=f'{self.identifier}', stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                self.subprocesses.append(p)
+                res = p.wait()
+                channel.sendall(struct.pack('>I', res))
+
+            return True
+
+        except Exception:
+            global running
+            if running:
+                print("ERROR: Unknown error")
+            return False
+
+class Connection(threading.Thread):
+    def __init__(self, sock, addr, host_key, authorized):
+        threading.Thread.__init__(self)
+        self.sock = sock
+        self.addr = addr
+        self.host_key = host_key
+        self.authorized = authorized
+        self.running = False
+        print("Connection from", addr)
+
+    def stop(self):
+        self.server.event.set()
+        self.server.stop()
+
+    def clean(self):
+        pass
+
+    def run(self):
+        transport = paramiko.Transport(self.sock)
+        transport.set_gss_host(socket.getfqdn(""))
+        transport.load_server_moduli()
+        transport.add_server_key(self.host_key)
+        server = SSHServer(self.authorized)
+        transport.start_server(server=server)
+        self.server = server
+        while not server.event.is_set():
+            if not transport.is_alive():
+                print("Connection", self.addr, "is broken from other end")
+                server.stop()
+                break
+            time.sleep(0.2)
+        else:
+            print("Connection", self.addr, "closed")
+        if server.identifier!='':
+            pass
+            shutil.rmtree(server.identifier, True)
+        transport.close()
+
+def print_help():
+    print("Unified FPGA synthesizer frontend - remote execution server\r\n(c) Joppe Blondel - 2022\r\n")
+    print(f"Usage: {sys.argv[0]} [ OPTIONS ] host port privkey pubkey authorized_hosts")
+    print("")
+    print("Options:")
+    print("  -h                 Show this help message")
+
+if __name__=="__main__":
+    # Parse arguments
+    i = 1
+    host = ''
+    port = ''
+    pubkey = ''
+    privkey = ''
+    authorized_f = ''
+    while i<len(sys.argv):
+        if sys.argv[i]=='-h':
+            print_help()
+            exit(0)
+        else:
+            if host=='':
+                host = sys.argv[i]
+            elif port=='':
+                port = sys.argv[i]
+            elif privkey=='':
+                privkey = sys.argv[i]
+            elif pubkey=='':
+                pubkey = sys.argv[i]
+            elif authorized_f=='':
+                authorized_f = sys.argv[i]
+        i += 1
+
+    signal.signal(signal.SIGINT, sighandler)
+
+    # Load SSH settings
+    host_key = paramiko.RSAKey(filename=privkey)
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock.bind((host, int(port)))
+    sock.settimeout(0.2)
+    sock.listen(100)
+    # Get authorized hosts
+    with open(authorized_f, 'r') as f:
+        authorized = f.read().split('\n')
+
+    running = True
+    while running:
+        try:
+            client, addr = sock.accept()
+            conn = Connection(client, addr, host_key, authorized)
+            conn.start()
+            threads.append(conn)
+        except TimeoutError as e:
+            pass
+
+    for t in threads:
+        t.join()
+        t.clean()