split up files, add release script

.gitignore

@@ -4,3 +4,5 @@ config.yaml
 0xf44ee3942e7dgendata.csv
 0xf44ee3942e7agendata.csv
 admin-key.ppk
+token.txt
+*.zip

ippigeon.py

@@ -1,15 +1,17 @@
-import taskbartool
+
-import os
+import sys
 import subprocess
 from multiprocessing import Process, Manager, Pool, TimeoutError, freeze_support, active_children
 from sys import platform
 from time import sleep
-import time
-import csv
 import uuid 
-import sys
 import yaml
-from fabric import Connection
+from util import find_data_file
+from util import fprint
+import taskbartool
+import util
+import netstat
+import ssh
 
 displaydata = None
 settings = None
@@ -38,140 +40,29 @@ if win32:
     _, username = res.strip().rsplit("\n", 1)
     userid, sysdom = username.rsplit("\\", 1)
 
-def find_data_file(filename):
-    if getattr(sys, "frozen", False):
-        # The application is frozen
-        datadir = os.path.dirname(sys.executable)
-    else:
-        # The application is not frozen
-        # Change this bit to match where you store your data files:
-        datadir = os.path.dirname(__file__)
-    return os.path.join(datadir, filename)
-
-def run_ps(cmd):
-    if win32:
-        startupinfo = subprocess.STARTUPINFO()
-        #print("DICKS")
-        #if not getattr(sys, "frozen", False):
-        #    print("test")
-        #    
-        #completed = subprocess.run(["powershell", "-Command", cmd], capture_output=True, startupinfo=startupinfo)
-        #else:
-        #    print("alt")
-        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW    # , "-WindowStyle", "hidden"
-        completed = subprocess.run(["powershell", "-Command", cmd], capture_output=True, startupinfo=startupinfo)
-        #completed = subprocess.run(["powershell", "-WindowStyle", "hidden", "-Command", cmd], capture_output=True, startupinfo=startupinfo)
-    
-        return completed
-
-def setup_child():
-    sys.stdout = Logger(filename=find_data_file("output.log"))
-    sys.stderr = Logger(filename=find_data_file("output.log"))
-
-def netstat():
-    setup_child()
-    print("netstat started")
-    if win32:
-        data = run_ps("netstat -n -o -b")
-        return data
-
 def netstat_done(res):
-    print("netstat done")
+    fprint("netstat done, processing")
-    procdata_res = pool.apply_async(process_netstat, (res,), callback=process_done)
+    procdata_res = pool.apply_async(netstat.process, (res,), callback=process_done)
-    #process_netstat(res)
+    #netstat.process(res)
-    #print(procdata_res.get())
-    #netdata_res = pool.apply_async(netstat)
 
 def process_done(res):
-    print("uploading to sftp...")
+    fprint("uploading to sftp...")
-    sftp_connect(4)
+    ssh.sftp_send_data(res, config, datafile)
-    #procdata_res = pool.apply_async(sftp_connect, (res,))
+    #procdata_res = pool.apply_async(ssh.sftp_send_data, (res, config, datafile))
     
-def sftp_connect(res):
-    print("Sending data over SFTP")
-    c = Connection(host=config['sftp']['host'], user=config['sftp']['user'], port=config['sftp']['port'], connect_kwargs={"key_filename": find_data_file(config['sftp']['keyfile']),})
-    c.put(find_data_file(datafile), remote=config['sftp']['filepath']['send'])
-    command = 'ls ' + config['sftp']['filepath']['send']
-    c.run(command)
-
 
 
-    #cnopts = pysftp.CnOpts(knownhosts=find_data_file('known_hosts'))
-    
-    #with pysftp.Connection(config['sftp']['host'], username=config['sftp']['user'], private_key=find_data_file(config['sftp']['keyfile']), cnopts=cnopts) as sftp:
-        #with sftp.cd(config['sftp']['filepath']['send']):
-            #sftp.put(find_data_file(datafile))
-
-def process_netstat(data):
-    setup_child()
-    print("netstat processing")
-    if win32:
-        #output = data.stdout
-        #print(output)
-        output = data.stdout.decode().split('\r\n') # split stdout into lines
-        #print(output)
-        if output[0].find("The requested operation requires elevation.") >= 0:
-            #print("test3")
-            raise PermissionError("Unable to acquire netstat data without admin!")
-        #print("test2")
-        output2 = list()
-        output2.append([sysid, userid, sysdom, int( time.time() )]) # add metadata
-        #print(output2)
-        procname = ""
-        """for x in range(4, len(output)):
-                tmp = output[x].split(" ")
-                print(tmp)
-                tmp = [i for i in output[x] if i]
-                print(tmp)
-                print(len(tmp))
-                if len(len(tmp) == 1):
-                    procname = tmp[0]
-                    print(x)
-                else:
-                    print(x)
-                    output2[x] = list()
-                    output2[x].append(procname)
-                    output2[x].append(output[x].split(" "))
-                    output2[x] = [i for i in output2[x] if i]
-        output2 = [i for i in output2 if i]
-        print(output2)"""
-        x = len(output) - 1 # start at the end because filename comes after connection
-        procname = "Unknown" # if the very last connection happens to have no file (yes, it's possible), we can say unknown
-        while x > 3:
-            string = output[x]
-            #print("LINE: ", string)
-            string_split = string.split(" ")
-            string_split = [i for i in string_split if i]
-            if string.find("Can not obtain ownership information") >= 0: # Higher privilige than us, must be system
-                procname = "Windows System"
-            elif string.find("]") >= 0 and string.find("[") == 1: # generic [file.exe]
-                procname = string[2:-1]
-            elif len(string_split) == 5: # actual netstat line
-                tmp = [procname,] # add executable name first
-                tmp.extend(string.split(" "))
-                tmp = [i for i in tmp if i]
-                #print(tmp)
-                output2.append(tmp)
-            #else: # In case of an extra line above file, or an empty line, ignore it
-                #print("Garbage data", string)
-            x = x - 1
-        #output2 = output2[2:]
-        #print(output2)
-        with open(find_data_file(datafile), "w", newline="") as f:
-            writer = csv.writer(f)
-            writer.writerows(output2)
-        print("done creating csv")
 
 def killall():
     kids = active_children()
     for kid in kids:
         kid.kill()
-    print("Every child has been killed")
+    fprint("Every child has been killed")
     os.kill(os.getpid(), 9) # dirty kill of self
 
 def mainloop(pool):
     # worker pool: netstat, netstat cleanup, upload, download, ui tasks
-    print("start loop")
+    fprint("start loop")
     
     global netdata_res
     global procdata_res
@@ -185,8 +76,9 @@ def mainloop(pool):
     if netdata_res is None or netdata_res.ready():
         #rawdata = netdata_res.get()
         #procdata_res = pool.apply_async(process_netstat, (rawdata))
-        print("netstat starting")
+        fprint("netstat starting")
-        netdata_res = pool.apply_async(netstat, callback=netstat_done)
+        netdata_res = pool.apply_async(netstat.start, callback=netstat_done)
+        #fprint(netdata_res.successful())
     sleep(interval)
 
 class Logger(object):
@@ -198,7 +90,10 @@ class Logger(object):
         self.log.write(message)
         #close(filename)
         #self.log = open(filename, "a")
-        self.terminal.write(message)
+        try:
+            self.terminal.write(message)
+        except:
+            sleep(0)
         
     def flush(self):
         print("")

netstat.py

@@ -0,0 +1,80 @@
+from util import find_data_file
+from util import setup_child
+from util import fprint
+from util import run_ps
+from util import win32
+import util
+import time
+import csv
+
+
+
+def process(data):
+    setup_child()
+    fprint("netstat processing")
+    if win32:
+        #output = data.stdout
+        #print(output)
+        output = data.stdout.decode().split('\r\n') # split stdout into lines
+        #print(output)
+        if output[0].find("The requested operation requires elevation.") >= 0:
+            #print("test3")
+            raise PermissionError("Unable to acquire netstat data without admin!")
+        #print("test2")
+        output2 = list()
+        output2.append([util.sysid, util.userid, util.sysdom, util.time()]) # add metadata
+        #print(output2)
+        procname = ""
+        """for x in range(4, len(output)):
+                tmp = output[x].split(" ")
+                print(tmp)
+                tmp = [i for i in output[x] if i]
+                print(tmp)
+                print(len(tmp))
+                if len(len(tmp) == 1):
+                    procname = tmp[0]
+                    print(x)
+                else:
+                    print(x)
+                    output2[x] = list()
+                    output2[x].append(procname)
+                    output2[x].append(output[x].split(" "))
+                    output2[x] = [i for i in output2[x] if i]
+        output2 = [i for i in output2 if i]
+        print(output2)"""
+        x = len(output) - 1 # start at the end because filename comes after connection
+        procname = "Unknown" # if the very last connection happens to have no file (yes, it's possible), we can say unknown
+        while x > 3:
+            string = output[x]
+            #print("LINE: ", string)
+            string_split = string.split(" ")
+            string_split = [i for i in string_split if i]
+            if string.find("Can not obtain ownership information") >= 0: # Higher privilige than us, must be system
+                procname = "Windows System"
+            elif string.find("]") >= 0 and string.find("[") == 1: # generic [file.exe]
+                procname = string[2:-1]
+            elif len(string_split) == 5: # actual netstat line
+                tmp = [procname,] # add executable name first
+                tmp.extend(string.split(" "))
+                tmp = [i for i in tmp if i]
+                #print(tmp)
+                output2.append(tmp)
+            #else: # In case of an extra line above file, or an empty line, ignore it
+                #print("Garbage data", string)
+            x = x - 1
+        #output2 = output2[2:]
+        #print(output2)
+        with open(find_data_file(util.datafile), "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerows(output2)
+        fprint("done creating csv")
+
+def start():
+    fprint("netstat started")
+    setup_child()
+    
+    if win32:
+        data = run_ps("netstat -n -o -b")
+        fprint("data acquired")
+        return data
+

release.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -xeuo pipefail
+
+TOKEN=$(< token.txt)
+git commit -a
+
+COMMIT=$(git log | head -n 1 | cut -d' ' -f2)
+echo $COMMIT
+export PATH=$PATH:"C:\Program Files\7-Zip"
+
+7z a -r release-$COMMIT.zip ./build/exe.win-amd64-3.10/*
+
+DATA='{
+  "body": "Autogenerated release",
+  "draft": true,
+  "name": "Development release",
+  "prerelease": true,
+  "tag_name": "testing",
+  "target_commitish": "'$COMMIT'"
+}'
+OUT=$(curl -X 'POST' \
+  'https://git.deck.sh/api/v1/repos/Interfaz/ff/releases?token='$TOKEN'' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d "$DATA")
+
+ID=$(echo $OUT | cut -d',' -f 1 | cut -d':' -f 2)
+
+OUT=curl -X 'POST' \
+  'https://git.deck.sh/api/v1/repos/Interfaz/ff/releases/'$ID'/assets?token='$TOKEN'' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: multipart/form-data' \
+  -F 'attachment=@'release-$COMMIT.zip';type=application/x-zip-compressed'
+

setup.py

@@ -1,13 +1,15 @@
 import sys
 from cx_Freeze import setup, Executable
 
+debug = True
+debug = not debug
 # Dependencies are automatically detected, but it might need fine tuning.
 # "packages": ["os"] is used as example only
-build_exe_options = {"packages": ["os"], "excludes": ["tkinter"], "include_msvcr": True, "include_files": ["icon.png", "IPPigeon.lnk", "config.yml", "known_hosts"], "optimize": 2}
+build_exe_options = {"packages": ["os"], "excludes": ["tkinter"], "include_msvcr": True, "include_files": ["icon.png", "IPPigeon.lnk", "config.yml", "keyfile-admin.pem"], "optimize": 2}
 
 # base="Win32GUI" should be used only for Windows GUI app
 base = None
-if sys.platform == "win32":
+if sys.platform == "win32" and not debug:
     base = "Win32GUI"
 
 setup(

ssh.py

@@ -0,0 +1,14 @@
+from fabric import Connection
+from util import find_data_file
+from util import setup_child
+from util import fprint
+
+def sftp_send_data(res, config, filename):
+    setup_child()
+    fprint("Connecting over SSH to " + config['sftp']['host'])
+    c = Connection(host=config['sftp']['host'], user=config['sftp']['user'], port=config['sftp']['port'], connect_kwargs={"key_filename": find_data_file(config['sftp']['keyfile']),})
+    fprint("Sending data over SFTP: " + filename)
+    fprint(c.put(find_data_file(filename), remote=config['sftp']['filepath']['send']))
+    fprint("Data sent over SFTP sucessfully")
+    #command = 'ls ' + config['sftp']['filepath']['send']
+    #fprint(c.run(command))

taskbartool.py

@@ -5,6 +5,8 @@ import os
 from time import sleep
 from sys import platform
 import sys
+from util import find_data_file
+from util import fprint
 
 TRAY_TOOLTIP = 'IP Pigeon' 
 
@@ -13,16 +15,6 @@ settings = None
 
 killme = False
 
-def find_data_file(filename):
-    if getattr(sys, "frozen", False):
-        # The application is frozen
-        datadir = os.path.dirname(sys.executable)
-    else:
-        # The application is not frozen
-        # Change this bit to match where you store your data files:
-        datadir = os.path.dirname(__file__)
-    return os.path.join(datadir, filename)
-    
 def create_menu_item(menu, label, func):
     item = wx.MenuItem(menu, -1, label)
     menu.Bind(wx.EVT_MENU, func, id=item.GetId())
@@ -48,7 +40,7 @@ class TaskBarIcon(wx.adv.TaskBarIcon):
         self.SetIcon(icon, TRAY_TOOLTIP)
 
     def on_left_down(self, event):      
-        print ('Tray icon was left-clicked.')
+        fprint ('Tray icon was left-clicked.')
 
     def on_open(self, event):
         foreground()

util.py

@@ -0,0 +1,98 @@
+import inspect
+import sys
+import subprocess
+import os
+from sys import platform
+import time as t
+import uuid
+
+win32 = platform == "win32"
+linux = platform == "linux" or platform == "linux2"
+macos = platform == "darwin"
+datafile = ""
+
+if win32:
+    sysid = hex(uuid.getnode())
+    datafile += sysid
+    datafile += "gendata.csv"
+    # Python is running as Administrator (so netstat can get filename, to block, etc), 
+    # so we use this to see who is actually logged in
+    # it's very hacky
+    startupinfo = subprocess.STARTUPINFO()
+    #if not getattr(sys, "frozen", False):
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW # hide powershell window
+    res = subprocess.check_output(["WMIC", "ComputerSystem", "GET", "UserName"], universal_newlines=True, startupinfo=startupinfo)
+    _, username = res.strip().rsplit("\n", 1)
+    userid, sysdom = username.rsplit("\\", 1)
+
+def time():
+    return int(t.time())
+
+def fprint(msg):
+    #if not getattr(sys, "frozen", False):
+    setup_child()
+    try:
+        frm = inspect.stack()[1]
+        
+        mod = inspect.getmodule(frm[0])
+        print('[' + mod.__name__ + ":" + frm.function + ']:', msg)
+    except Exception as e:
+        try:
+            print('[????:' + frm.function + ']:', msg)
+        except:
+            print('[????]:', msg)
+        
+        
+   # else:
+        #print(msg)
+
+def find_data_file(filename):
+    if getattr(sys, "frozen", False):
+        # The application is frozen
+        datadir = os.path.dirname(sys.executable)
+    else:
+        # The application is not frozen
+        # Change this bit to match where you store your data files:
+        datadir = os.path.dirname(__file__)
+    return os.path.join(datadir, filename)
+
+def run_ps(cmd):
+    fprint("init PS")
+    if win32:
+        startupinfo = subprocess.STARTUPINFO()
+        #print("DICKS")
+        #if not getattr(sys, "frozen", False):
+        #    print("test")
+        #    
+        #completed = subprocess.run(["powershell", "-Command", cmd], capture_output=True, startupinfo=startupinfo)
+        #else:
+        #    print("alt")
+        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW    # , "-WindowStyle", "hidden"
+        fprint("running PS command: " + cmd)
+        completed = subprocess.run(["powershell", "-Command", cmd], capture_output=True, startupinfo=startupinfo)
+        fprint("ran PS command sucessfully")
+        #completed = subprocess.run(["powershell", "-WindowStyle", "hidden", "-Command", cmd], capture_output=True, startupinfo=startupinfo)
+    
+        return completed
+
+def setup_child():
+    if not getattr(sys, "frozen", False):
+        sys.stdout = Logger(filename=find_data_file("output.log"))
+        sys.stderr = Logger(filename=find_data_file("output.log"))
+
+class Logger(object):
+    def __init__(self, filename="output.log"):
+        self.log = open(filename, "a")
+        self.terminal = sys.stdout
+
+    def write(self, message):
+        self.log.write(message)
+        #close(filename)
+        #self.log = open(filename, "a")
+        try:
+            self.terminal.write(message)
+        except:
+            sleep(0)
+        
+    def flush(self):
+        print("")