From 66fe15eb68e9ea2169eee5ac51e1cca1d1f7a0a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Fri, 17 May 2019 22:20:12 -0400
Subject: [PATCH 01/17] MOved to os.open()

---
 SerialPort.py | 45 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index c99691c3..41ae9d7a 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -8,7 +8,7 @@
 
 class SerialPort:
     """SerialPort class with basic application-level protocol functions to write strings and read strings"""
-    port = None
+    fd = None
     bsdPath = None
     vendorId = None
     productId = None
@@ -24,21 +24,44 @@ def drain(self):
         return None
 
     def open(self):
-        try:
-            self.port = open(self.bsdPath, "r+")
-        except OSError as msg:
-            self.port = None
-            raise SerialException(msg.errno, "could not open port {}: {}".format(self.port, msg))
+        self.fd = os.open(self.bsdPath, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+
+        if self.fd != -1:
+            try:
+                value = fcntl.ioctl(self.fd, fcntl.F_SETFL, fcntl.ioctl(self.fd, fcntl.F_GETFL,0))
+                if  value & ~os.O_NONBLOCK == -1:
+                    raise ValueError(-1)
+            except OSError as msg:
+                print(msg)
 
     def close(self):
-        self.port.close()
+        os.close(self.fd)
+
+    def readData(self, length):
+        data = os.read(self.fd, length)
+        return data
+
+    def writeData(self, data):
+        nBytesWritten = os.write(self.fd, data)
+        return nBytesWritten
+
+    def readString(self):
+        byte = None
+        data = bytearray(0)
+        while ( byte != "\n"):
+            byte = self.readData(1)
+            if byte == b'':
+                break
+            else:
+                data += byte
+
+        string = data.decode(encoding='utf-8')
 
-    def readString(self, matching=".*"):
-        string = self.port.readline()
         return string
 
     def writeString(self, string):
-        return self.port.write(string)
+        data = bytearray(string, "utf-8")
+        return os.write(self.fd, data)
         
 
 
@@ -46,5 +69,5 @@ def writeString(self, string):
     port = SerialPort("/dev/null")
     port.open()
     port.writeString("abcd")
-    string = port.readString()
+    print(port.readData(1))
     port.close()

From c4935bf10ad1f8ac1e375f461fc92e7e29376b0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sat, 18 May 2019 00:52:07 -0400
Subject: [PATCH 02/17] Attempts with POSIX ports.

---
 SerialPort.py | 38 ++++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 41ae9d7a..f482b217 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -5,6 +5,8 @@
 import struct
 import sys
 import termios
+import array
+import math
 
 class SerialPort:
     """SerialPort class with basic application-level protocol functions to write strings and read strings"""
@@ -24,21 +26,29 @@ def drain(self):
         return None
 
     def open(self):
-        self.fd = os.open(self.bsdPath, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+        self.fd = os.open(self.bsdPath, os.O_RDWR | os.O_NOCTTY)
 
-        if self.fd != -1:
-            try:
-                value = fcntl.ioctl(self.fd, fcntl.F_SETFL, fcntl.ioctl(self.fd, fcntl.F_GETFL,0))
-                if  value & ~os.O_NONBLOCK == -1:
-                    raise ValueError(-1)
-            except OSError as msg:
-                print(msg)
+        # flag = fcntl.fcntl(self.fd, fcntl.F_GETFL,0)
+        # fcntl.fcntl(self.fd, fcntl.F_SETFL, flag & ~os.O_NONBLOCK)
 
     def close(self):
         os.close(self.fd)
 
+    def bytesAvailable(self):
+        bytes = bytearray(4)
+        arg = 0
+        n = fcntl.fcntl(self.fd, termios.FIONREAD, arg)
+
+        print(n, arg)
+        return bytes
+
     def readData(self, length):
-        data = os.read(self.fd, length)
+        bytesAvailable = self.bytesAvailable()
+
+        if length <= bytesAvailable:
+            data = os.read(self.fd, length)
+        else:
+            data = os.read(self.fd, bytesAvailable)
         return data
 
     def writeData(self, data):
@@ -48,9 +58,9 @@ def writeData(self, data):
     def readString(self):
         byte = None
         data = bytearray(0)
-        while ( byte != "\n"):
+        while ( byte != b''):
             byte = self.readData(1)
-            if byte == b'':
+            if byte == b'\n':
                 break
             else:
                 data += byte
@@ -66,8 +76,8 @@ def writeString(self, string):
 
 
 if __name__ == "__main__":
-    port = SerialPort("/dev/null")
+    port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
     port.open()
-    port.writeString("abcd")
-    print(port.readData(1))
+    port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd")
+    print(port.readData(8))
     port.close()

From 51b2a2d7d550220865e43e376ba6e268e4aed178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sat, 18 May 2019 00:52:22 -0400
Subject: [PATCH 03/17] pySerial port

---
 SerialPort.py | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index f482b217..f2811e2a 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -7,10 +7,11 @@
 import termios
 import array
 import math
+import serial
 
 class SerialPort:
     """SerialPort class with basic application-level protocol functions to write strings and read strings"""
-    fd = None
+    port = None
     bsdPath = None
     vendorId = None
     productId = None
@@ -26,10 +27,7 @@ def drain(self):
         return None
 
     def open(self):
-        self.fd = os.open(self.bsdPath, os.O_RDWR | os.O_NOCTTY)
-
-        # flag = fcntl.fcntl(self.fd, fcntl.F_GETFL,0)
-        # fcntl.fcntl(self.fd, fcntl.F_SETFL, flag & ~os.O_NONBLOCK)
+        self.port = Serial(self.bsdPath, 19200, timeout=1)
 
     def close(self):
         os.close(self.fd)
@@ -43,13 +41,7 @@ def bytesAvailable(self):
         return bytes
 
     def readData(self, length):
-        bytesAvailable = self.bytesAvailable()
-
-        if length <= bytesAvailable:
-            data = os.read(self.fd, length)
-        else:
-            data = os.read(self.fd, bytesAvailable)
-        return data
+        data = os.read(self.fd, length)
 
     def writeData(self, data):
         nBytesWritten = os.write(self.fd, data)

From 59c9c483a51829d7b61589b96d4f26ae5bdfa9ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sat, 18 May 2019 01:14:36 -0400
Subject: [PATCH 04/17] Update SerialPort.py

Actually operationnal thanks to pyserial.
---
 SerialPort.py | 49 ++++++++++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 21 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index f2811e2a..860259ef 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -7,7 +7,8 @@
 import termios
 import array
 import math
-import serial
+import serial #pyserial
+import re
 
 class SerialPort:
     """SerialPort class with basic application-level protocol functions to write strings and read strings"""
@@ -20,31 +21,27 @@ class SerialPort:
     def __init__(self, bsdPath = None, vendorId = None, productId = None, serialNumber = None):
         self.bsdPath = bsdPath
 
-    def flush(self):
-        return None
-
-    def drain(self):
-        return None
-
     def open(self):
-        self.port = Serial(self.bsdPath, 19200, timeout=1)
+        self.port = serial.Serial(self.bsdPath, 19200, timeout=1)
 
     def close(self):
-        os.close(self.fd)
+        self.port.close()
 
     def bytesAvailable(self):
-        bytes = bytearray(4)
-        arg = 0
-        n = fcntl.fcntl(self.fd, termios.FIONREAD, arg)
+        return self.port.inwaiting()
 
-        print(n, arg)
-        return bytes
+    def flush(self):
+        return None
+
+    def drain(self):
+        return None
 
     def readData(self, length):
-        data = os.read(self.fd, length)
+        data = self.port.read(length)
+        return data
 
     def writeData(self, data):
-        nBytesWritten = os.write(self.fd, data)
+        nBytesWritten = self.port.write(data)
         return nBytesWritten
 
     def readString(self):
@@ -52,10 +49,9 @@ def readString(self):
         data = bytearray(0)
         while ( byte != b''):
             byte = self.readData(1)
+            data += byte
             if byte == b'\n':
                 break
-            else:
-                data += byte
 
         string = data.decode(encoding='utf-8')
 
@@ -63,13 +59,24 @@ def readString(self):
 
     def writeString(self, string):
         data = bytearray(string, "utf-8")
-        return os.write(self.fd, data)
+        return self.port.write(data)
+
+    def writeReadMatch(self, string, replyPattern):
+        self.writeString(string)
+        reply = self.readString()
+        match = re.search(replyPattern, string)
+        
+        return match is not None
+
+
         
 
 
 if __name__ == "__main__":
     port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
     port.open()
-    port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd")
-    print(port.readData(8))
+    port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\n")
+    print(port.readString())
+    print(port.writeReadMatch(string="abcd\n", replyPattern="abcd"))
+    print(port.writeReadMatch(string="abcd\n", replyPattern="abcad"))
     port.close()

From edc3604b58db02d23d417c8534a71683034b7ebe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sat, 18 May 2019 01:20:11 -0400
Subject: [PATCH 05/17] writeRead commands

---
 SerialPort.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 860259ef..55a1b606 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -61,14 +61,24 @@ def writeString(self, string):
         data = bytearray(string, "utf-8")
         return self.port.write(data)
 
-    def writeReadMatch(self, string, replyPattern):
+    def writeStringExpectMatchingString(self, string, replyPattern):
         self.writeString(string)
         reply = self.readString()
         match = re.search(replyPattern, string)
         
         return match is not None
 
+    def writeStringReadFirstMatchingGroup(self, string, replyPattern):
+        self.writeString(string)
+        reply = self.readString()
+        match = re.search(replyPattern, string)
+        
+        if match is not None:
+            groups = match.groups()
+            if len(groups) >= 1:
+                return match.group(1)
 
+        return None            
         
 
 
@@ -77,6 +87,7 @@ def writeReadMatch(self, string, replyPattern):
     port.open()
     port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\n")
     print(port.readString())
-    print(port.writeReadMatch(string="abcd\n", replyPattern="abcd"))
-    print(port.writeReadMatch(string="abcd\n", replyPattern="abcad"))
+    print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcd"))
+    print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcad"))
+    print(port.writeStringReadFirstMatchingGroup(string="abcd\n", replyPattern="ab(cd)"))
     port.close()

From 2907fb3553d801527239389549fe588a2b9dc4a8 Mon Sep 17 00:00:00 2001
From: Daniel Cote <dccote@novajo.ca>
Date: Sun, 19 May 2019 13:46:56 -0400
Subject: [PATCH 06/17] First debug port

---
 SerialPort.py | 69 ++++++++++++++++++++++++++++++++++++++-------------
 testDebug.py  | 33 ++++++++++++++++++++++++
 2 files changed, 85 insertions(+), 17 deletions(-)
 create mode 100644 testDebug.py

diff --git a/SerialPort.py b/SerialPort.py
index 55a1b606..39a07d5b 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -7,7 +7,7 @@
 import termios
 import array
 import math
-import serial #pyserial
+import serial
 import re
 
 class SerialPort:
@@ -42,7 +42,8 @@ def readData(self, length):
 
     def writeData(self, data):
         nBytesWritten = self.port.write(data)
-        return nBytesWritten
+        if nBytesWritten != len(data):
+            raise IOError("Not all bytes written to port")
 
     def readString(self):
         byte = None
@@ -54,40 +55,74 @@ def readString(self):
                 break
 
         string = data.decode(encoding='utf-8')
-
         return string
 
     def writeString(self, string):
         data = bytearray(string, "utf-8")
-        return self.port.write(data)
+        self.writeData(data)
 
     def writeStringExpectMatchingString(self, string, replyPattern):
         self.writeString(string)
         reply = self.readString()
         match = re.search(replyPattern, string)
-        
-        return match is not None
+        if match is None:
+            raise RuntimeError("No match")
+
+        return reply
 
     def writeStringReadFirstMatchingGroup(self, string, replyPattern):
+        groups = self.writeStringReadMatchingGroups(string, replyPattern)
+        if len(groups) >= 1:
+            return match.group(1)
+        else:
+            raise RuntimeError("No match")
+
+    def writeStringReadMatchingGroups(self, string, replyPattern):
         self.writeString(string)
         reply = self.readString()
         match = re.search(replyPattern, string)
         
         if match is not None:
-            groups = match.groups()
-            if len(groups) >= 1:
-                return match.group(1)
+            return match.groups()
+        else:
+            raise RuntimeError("No match")
 
-        return None            
-        
+class DebugEchoSerialPort(SerialPort):
+    buffer = bytearray()
+
+    def open(self):
+        return
+
+    def close(self):
+        return
 
+    def readData(self, length):
+        data = bytearray()
+        for i in range(0,length):
+            byte = self.buffer.pop(0)
+            print(i, byte)
+            data.append(byte)
+
+        return data
+
+    def writeData(self, data):
+        self.buffer.extend(data)
+        print(self.buffer)
+        return len(data)
 
 if __name__ == "__main__":
-    port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
+    port = DebugEchoSerialPort()
     port.open()
-    port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\n")
-    print(port.readString())
-    print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcd"))
-    print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcad"))
-    print(port.writeStringReadFirstMatchingGroup(string="abcd\n", replyPattern="ab(cd)"))
     port.close()
+
+    # port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
+    # port.open()
+    # port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\n")
+    # print(port.readString())
+    # print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcd"))
+    # print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcad"))
+    # print(port.writeStringReadFirstMatchingGroup(string="abcd\n", replyPattern="ab(cd)"))
+    # port.close()
+
+
+
diff --git a/testDebug.py b/testDebug.py
new file mode 100644
index 00000000..f2829b06
--- /dev/null
+++ b/testDebug.py
@@ -0,0 +1,33 @@
+import unittest
+from SerialPort import *
+
+payload = b'1234'
+
+class TestSerialPortWithEcho(unittest.TestCase):
+    serialPort = DebugEchoSerialPort()
+
+    def setUp(self):
+        self.assertIsNotNone(self.serialPort)
+        self.serialPort.open()
+
+    def tearDown(self):
+        self.serialPort.close()
+
+    def testCreate(self):
+        self.assertIsNotNone(self.serialPort)
+
+    def testWrite(self):
+        nBytes = self.serialPort.writeData(payload)
+        self.assertTrue(nBytes == len(payload))
+
+    def testWriteReadEcho(self):
+        nBytes = self.serialPort.writeData(payload)
+        self.assertTrue(nBytes == len(payload))
+
+        data = self.serialPort.readData(length=len(payload))
+        self.assertTrue(data == payload)
+
+
+
+if __name__ == '__main__':
+    unittest.main()

From 9b7924a82e72b19b0babdb0ca48056f08a4cd9b6 Mon Sep 17 00:00:00 2001
From: Daniel Cote <dccote@novajo.ca>
Date: Sun, 19 May 2019 16:17:39 -0400
Subject: [PATCH 07/17] Echo tests

---
 SerialPort.py | 20 ++++++++++----------
 testDebug.py  | 35 ++++++++++++++++++++++++++---------
 tests.py      |  2 ++
 3 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 39a07d5b..481539c7 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -10,15 +10,18 @@
 import serial
 import re
 
+
 class SerialPort:
-    """SerialPort class with basic application-level protocol functions to write strings and read strings"""
+    """SerialPort class with basic application-level protocol 
+    functions to write strings and read strings"""
     port = None
     bsdPath = None
     vendorId = None
     productId = None
     serialNumber = None
 
-    def __init__(self, bsdPath = None, vendorId = None, productId = None, serialNumber = None):
+    def __init__(self, bsdPath=None, vendorId=None,
+                 productId=None, serialNumber=None):
         self.bsdPath = bsdPath
 
     def open(self):
@@ -48,7 +51,7 @@ def writeData(self, data):
     def readString(self):
         byte = None
         data = bytearray(0)
-        while ( byte != b''):
+        while (byte != b''):
             byte = self.readData(1)
             data += byte
             if byte == b'\n':
@@ -81,12 +84,13 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
         self.writeString(string)
         reply = self.readString()
         match = re.search(replyPattern, string)
-        
+
         if match is not None:
             return match.groups()
         else:
             raise RuntimeError("No match")
 
+
 class DebugEchoSerialPort(SerialPort):
     buffer = bytearray()
 
@@ -98,18 +102,17 @@ def close(self):
 
     def readData(self, length):
         data = bytearray()
-        for i in range(0,length):
+        for i in range(0, length):
             byte = self.buffer.pop(0)
-            print(i, byte)
             data.append(byte)
 
         return data
 
     def writeData(self, data):
         self.buffer.extend(data)
-        print(self.buffer)
         return len(data)
 
+
 if __name__ == "__main__":
     port = DebugEchoSerialPort()
     port.open()
@@ -123,6 +126,3 @@ def writeData(self, data):
     # print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcad"))
     # print(port.writeStringReadFirstMatchingGroup(string="abcd\n", replyPattern="ab(cd)"))
     # port.close()
-
-
-
diff --git a/testDebug.py b/testDebug.py
index f2829b06..25a526f1 100644
--- a/testDebug.py
+++ b/testDebug.py
@@ -1,33 +1,50 @@
 import unittest
+from serial import *
 from SerialPort import *
 
 payload = b'1234'
 
-class TestSerialPortWithEcho(unittest.TestCase):
-    serialPort = DebugEchoSerialPort()
+class TestEchoPort(unittest.TestCase):
+    port = None
 
     def setUp(self):
-        self.assertIsNotNone(self.serialPort)
-        self.serialPort.open()
+        self.port = DebugEchoSerialPort()
+        self.assertIsNotNone(self.port)
+        self.port.open()
 
     def tearDown(self):
-        self.serialPort.close()
+        self.port.close()
 
     def testCreate(self):
-        self.assertIsNotNone(self.serialPort)
+        self.assertIsNotNone(self.port)
 
     def testWrite(self):
-        nBytes = self.serialPort.writeData(payload)
+        nBytes = self.port.writeData(payload)
         self.assertTrue(nBytes == len(payload))
 
     def testWriteReadEcho(self):
-        nBytes = self.serialPort.writeData(payload)
+        nBytes = self.port.writeData(payload)
         self.assertTrue(nBytes == len(payload))
 
-        data = self.serialPort.readData(length=len(payload))
+        data = self.port.readData(length=len(payload))
         self.assertTrue(data == payload)
 
+    def testWriteReadEchoSequence(self):
+        nBytes = self.port.writeData(payload)
+        nBytes = self.port.writeData(payload)
 
+        data = self.port.readData(length=len(payload))
+        self.assertTrue(data == payload)
+        data = self.port.readData(length=len(payload))
+        self.assertTrue(data == payload)
+
+    def testWriteReadEchoLarge(self):
+        for i in range(100):
+            nBytes = self.port.writeData(payload)
+
+        for i in range(100):
+            data = self.port.readData(length=len(payload))
+            self.assertTrue(data == payload)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests.py b/tests.py
index c2876374..76307df6 100644
--- a/tests.py
+++ b/tests.py
@@ -1,6 +1,7 @@
 import unittest
 from SerialPort import *
 
+
 class TestSerialPortWithEcho(unittest.TestCase):
     serialPort = SerialPort("/dev/cu.usbserial-DPB3LCH1")
 
@@ -14,5 +15,6 @@ def tearDown(self):
     def testCreate(self):
         self.assertIsNotNone(self.serialPort)
 
+
 if __name__ == '__main__':
     unittest.main()

From bdcc43516bcab3ee07f01d86f32cf8980e6e70f4 Mon Sep 17 00:00:00 2001
From: Daniel Cote <dccote@novajo.ca>
Date: Sun, 19 May 2019 16:52:25 -0400
Subject: [PATCH 08/17] more Tests for serrial and debug serrial port

---
 SerialPort.py | 91 ++++++++++++++++++++++++++++++++++++---------------
 testDebug.py  | 78 ++++++++++++++++++++++++++++++++-----------
 2 files changed, 123 insertions(+), 46 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 481539c7..a3e02581 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -9,20 +9,22 @@
 import math
 import serial
 import re
+from threading import Thread, RLock
 
+class CommunicationReadTimeout(serial.SerialException):
+    pass
 
 class SerialPort:
     """SerialPort class with basic application-level protocol 
     functions to write strings and read strings"""
     port = None
-    bsdPath = None
-    vendorId = None
-    productId = None
-    serialNumber = None
-
     def __init__(self, bsdPath=None, vendorId=None,
                  productId=None, serialNumber=None):
         self.bsdPath = bsdPath
+        self.vendorId = vendorId
+        self.productId = productId
+        self.serialNumber = serialNumber
+        self.lock = RLock()
 
     def open(self):
         self.port = serial.Serial(self.bsdPath, 19200, timeout=1)
@@ -40,29 +42,52 @@ def drain(self):
         return None
 
     def readData(self, length):
-        data = self.port.read(length)
+        self.lock.acquire()
+        try:
+            data = self.port.read(length)
+        finally:
+            self.lock.acquire()
+
         return data
 
     def writeData(self, data):
-        nBytesWritten = self.port.write(data)
-        if nBytesWritten != len(data):
-            raise IOError("Not all bytes written to port")
+        self.lock.acquire()
+        try:
+            nBytesWritten = self.port.write(data)
+            if nBytesWritten != len(data):
+                raise IOError("Not all bytes written to port")
+        finally:
+            self.lock.acquire()
+
+        return nBytesWritten
 
     def readString(self):
-        byte = None
-        data = bytearray(0)
-        while (byte != b''):
-            byte = self.readData(1)
-            data += byte
-            if byte == b'\n':
-                break
-
-        string = data.decode(encoding='utf-8')
+        self.lock.acquire()
+        try:
+            byte = None
+            data = bytearray(0)
+            while (byte != b''):
+                byte = self.readData(1)
+                data += byte
+                if byte == b'\n':
+                    break
+
+            string = data.decode(encoding='utf-8')
+        finally:
+            self.lock.release()
+
         return string
 
     def writeString(self, string):
-        data = bytearray(string, "utf-8")
-        self.writeData(data)
+        self.lock.acquire()
+        nBytes = 0
+        try:
+            data = bytearray(string, "utf-8")
+            nBytes = self.writeData(data)
+        finally:
+            self.lock.release()
+
+        return nBytes
 
     def writeStringExpectMatchingString(self, string, replyPattern):
         self.writeString(string)
@@ -92,7 +117,9 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
 
 
 class DebugEchoSerialPort(SerialPort):
-    buffer = bytearray()
+    def __init__(self):
+        self.buffer = bytearray()
+        super(DebugEchoSerialPort, self).__init__()
 
     def open(self):
         return
@@ -101,15 +128,27 @@ def close(self):
         return
 
     def readData(self, length):
-        data = bytearray()
-        for i in range(0, length):
-            byte = self.buffer.pop(0)
-            data.append(byte)
+        self.lock.acquire()
+        try:
+            data = bytearray()
+            for i in range(0, length):
+                if len(self.buffer) > 0:
+                    byte = self.buffer.pop(0)
+                    data.append(byte)
+                else:
+                    raise CommunicationReadTimeout("Unable to read data")
+        finally:
+            self.lock.release()
 
         return data
 
     def writeData(self, data):
-        self.buffer.extend(data)
+        self.lock.acquire()
+        try:
+            self.buffer.extend(data)
+        finally:
+            self.lock.release()
+
         return len(data)
 
 
diff --git a/testDebug.py b/testDebug.py
index 25a526f1..7b14f9d7 100644
--- a/testDebug.py
+++ b/testDebug.py
@@ -2,7 +2,8 @@
 from serial import *
 from SerialPort import *
 
-payload = b'1234'
+payloadData = b'1234'
+payloadString = '1234\n'
 
 class TestEchoPort(unittest.TestCase):
     port = None
@@ -18,33 +19,70 @@ def tearDown(self):
     def testCreate(self):
         self.assertIsNotNone(self.port)
 
-    def testWrite(self):
-        nBytes = self.port.writeData(payload)
-        self.assertTrue(nBytes == len(payload))
+    def testWriteData(self):
+        nBytes = self.port.writeData(payloadData)
+        self.assertTrue(nBytes == len(payloadData))
 
-    def testWriteReadEcho(self):
-        nBytes = self.port.writeData(payload)
-        self.assertTrue(nBytes == len(payload))
+    def testWriteDataReadEcho(self):
+        nBytes = self.port.writeData(payloadData)
+        self.assertTrue(nBytes == len(payloadData))
 
-        data = self.port.readData(length=len(payload))
-        self.assertTrue(data == payload)
+        data = self.port.readData(length=len(payloadData))
+        self.assertTrue(data == payloadData)
 
-    def testWriteReadEchoSequence(self):
-        nBytes = self.port.writeData(payload)
-        nBytes = self.port.writeData(payload)
+    def testWriteDataReadEchoSequence(self):
+        nBytes = self.port.writeData(payloadData)
+        nBytes = self.port.writeData(payloadData)
 
-        data = self.port.readData(length=len(payload))
-        self.assertTrue(data == payload)
-        data = self.port.readData(length=len(payload))
-        self.assertTrue(data == payload)
+        data = self.port.readData(length=len(payloadData))
+        self.assertTrue(data == payloadData)
+        data = self.port.readData(length=len(payloadData))
+        self.assertTrue(data == payloadData)
 
-    def testWriteReadEchoLarge(self):
+    def testWriteDataReadEchoLarge(self):
         for i in range(100):
-            nBytes = self.port.writeData(payload)
+            nBytes = self.port.writeData(payloadData)
 
         for i in range(100):
-            data = self.port.readData(length=len(payload))
-            self.assertTrue(data == payload)
+            data = self.port.readData(length=len(payloadData))
+            self.assertTrue(data == payloadData)
+
+    def testWriteString(self):
+        nBytes = self.port.writeString(payloadString)
+        self.assertTrue(nBytes == len(payloadString))
+
+    def testWriteStringReadEcho(self):
+        nBytes = self.port.writeString(payloadString)
+        self.assertTrue(nBytes == len(payloadString))
+
+        string = self.port.readString()
+        self.assertTrue(string == payloadString, "String {0}, payload:{1}".format(string, payloadString))
+
+    def testWriteStringReadEchoSequence(self):
+        nBytes = self.port.writeString(payloadString)
+        nBytes = self.port.writeString(payloadString)
+
+        string = self.port.readString()
+        self.assertTrue(string == payloadString)
+        string = self.port.readString()
+        self.assertTrue(string == payloadString)
+
+    def testWriteStringReadEchoLarge(self):
+        for i in range(100):
+            nBytes = self.port.writeString(payloadString)
+
+        for i in range(100):
+            string = self.port.readString()
+            self.assertTrue(string == payloadString)
+
+    def testTimeoutReadData(self):
+        with self.assertRaises(CommunicationReadTimeout) as context:
+            self.port.readData(1)
+
+        self.port.writeData(payloadData)
+        with self.assertRaises(CommunicationReadTimeout) as context:
+            self.port.readData(len(payloadData)+1)
+
 
 if __name__ == '__main__':
     unittest.main()

From 4ce51a3f997b1e2fead470c0436749c3aef8da5c Mon Sep 17 00:00:00 2001
From: Daniel Cote <dccote@novajo.ca>
Date: Sun, 19 May 2019 17:01:55 -0400
Subject: [PATCH 09/17] moved tests to base classe for debug and real device

---
 testDebug.py | 129 +++++++++++++++++++++++++++++----------------------
 1 file changed, 74 insertions(+), 55 deletions(-)

diff --git a/testDebug.py b/testDebug.py
index 7b14f9d7..48ac88d5 100644
--- a/testDebug.py
+++ b/testDebug.py
@@ -5,83 +5,102 @@
 payloadData = b'1234'
 payloadString = '1234\n'
 
-class TestEchoPort(unittest.TestCase):
-    port = None
+class BaseTestCases:
 
-    def setUp(self):
-        self.port = DebugEchoSerialPort()
-        self.assertIsNotNone(self.port)
-        self.port.open()
+    class TestEchoPort(unittest.TestCase):
+        port = None
 
-    def tearDown(self):
-        self.port.close()
+        def testCreate(self):
+            self.assertIsNotNone(self.port)
 
-    def testCreate(self):
-        self.assertIsNotNone(self.port)
-
-    def testWriteData(self):
-        nBytes = self.port.writeData(payloadData)
-        self.assertTrue(nBytes == len(payloadData))
-
-    def testWriteDataReadEcho(self):
-        nBytes = self.port.writeData(payloadData)
-        self.assertTrue(nBytes == len(payloadData))
-
-        data = self.port.readData(length=len(payloadData))
-        self.assertTrue(data == payloadData)
+        def testWriteData(self):
+            nBytes = self.port.writeData(payloadData)
+            self.assertTrue(nBytes == len(payloadData))
 
-    def testWriteDataReadEchoSequence(self):
-        nBytes = self.port.writeData(payloadData)
-        nBytes = self.port.writeData(payloadData)
+        def testWriteDataReadEcho(self):
+            nBytes = self.port.writeData(payloadData)
+            self.assertTrue(nBytes == len(payloadData))
 
-        data = self.port.readData(length=len(payloadData))
-        self.assertTrue(data == payloadData)
-        data = self.port.readData(length=len(payloadData))
-        self.assertTrue(data == payloadData)
+            data = self.port.readData(length=len(payloadData))
+            self.assertTrue(data == payloadData)
 
-    def testWriteDataReadEchoLarge(self):
-        for i in range(100):
+        def testWriteDataReadEchoSequence(self):
+            nBytes = self.port.writeData(payloadData)
             nBytes = self.port.writeData(payloadData)
 
-        for i in range(100):
+            data = self.port.readData(length=len(payloadData))
+            self.assertTrue(data == payloadData)
             data = self.port.readData(length=len(payloadData))
             self.assertTrue(data == payloadData)
 
-    def testWriteString(self):
-        nBytes = self.port.writeString(payloadString)
-        self.assertTrue(nBytes == len(payloadString))
+        def testWriteDataReadEchoLarge(self):
+            for i in range(100):
+                nBytes = self.port.writeData(payloadData)
 
-    def testWriteStringReadEcho(self):
-        nBytes = self.port.writeString(payloadString)
-        self.assertTrue(nBytes == len(payloadString))
+            for i in range(100):
+                data = self.port.readData(length=len(payloadData))
+                self.assertTrue(data == payloadData)
 
-        string = self.port.readString()
-        self.assertTrue(string == payloadString, "String {0}, payload:{1}".format(string, payloadString))
+        def testWriteString(self):
+            nBytes = self.port.writeString(payloadString)
+            self.assertTrue(nBytes == len(payloadString))
 
-    def testWriteStringReadEchoSequence(self):
-        nBytes = self.port.writeString(payloadString)
-        nBytes = self.port.writeString(payloadString)
+        def testWriteStringReadEcho(self):
+            nBytes = self.port.writeString(payloadString)
+            self.assertTrue(nBytes == len(payloadString))
 
-        string = self.port.readString()
-        self.assertTrue(string == payloadString)
-        string = self.port.readString()
-        self.assertTrue(string == payloadString)
+            string = self.port.readString()
+            self.assertTrue(string == payloadString, "String {0}, payload:{1}".format(string, payloadString))
 
-    def testWriteStringReadEchoLarge(self):
-        for i in range(100):
+        def testWriteStringReadEchoSequence(self):
+            nBytes = self.port.writeString(payloadString)
             nBytes = self.port.writeString(payloadString)
 
-        for i in range(100):
             string = self.port.readString()
             self.assertTrue(string == payloadString)
+            string = self.port.readString()
+            self.assertTrue(string == payloadString)
+
+        def testWriteStringReadEchoLarge(self):
+            for i in range(100):
+                nBytes = self.port.writeString(payloadString)
+
+            for i in range(100):
+                string = self.port.readString()
+                self.assertTrue(string == payloadString)
+
+        def testTimeoutReadData(self):
+            with self.assertRaises(CommunicationReadTimeout) as context:
+                self.port.readData(1)
+
+            self.port.writeData(payloadData)
+            with self.assertRaises(CommunicationReadTimeout) as context:
+                self.port.readData(len(payloadData)+1)
 
-    def testTimeoutReadData(self):
-        with self.assertRaises(CommunicationReadTimeout) as context:
-            self.port.readData(1)
+class TestDebugEchoPort(BaseTestCases.TestEchoPort):
+
+    def setUp(self):
+        self.port = DebugEchoSerialPort()
+        self.assertIsNotNone(self.port)
+        self.port.open()
 
-        self.port.writeData(payloadData)
-        with self.assertRaises(CommunicationReadTimeout) as context:
-            self.port.readData(len(payloadData)+1)
+    def tearDown(self):
+        self.port.close()
+
+
+class TestRealEchoPort(BaseTestCases.TestEchoPort):
+
+    def setUp(self):
+        try:
+            self.port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
+            self.assertIsNotNone(self.port)
+            self.port.open()
+        except:
+            self.fail("Unable to setUp serial port")
+
+
+    def tearDown(self):
+        self.port.close()
 
 
 if __name__ == '__main__':

From 1c3a275f2ea3e92bd5048d0c808679455e4156dc Mon Sep 17 00:00:00 2001
From: Daniel Cote <dccote@novajo.ca>
Date: Sun, 19 May 2019 17:16:46 -0400
Subject: [PATCH 10/17] New exception: Read No Match

MOre test with groups and failed matches
---
 SerialPort.py | 11 +++++----
 testDebug.py  | 64 +++++++++++++++++++++++++++++++++++++++++++--------
 2 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index a3e02581..7a2fb1fd 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -14,6 +14,9 @@
 class CommunicationReadTimeout(serial.SerialException):
     pass
 
+class CommunicationReadNoMatch(Exception):
+    pass
+
 class SerialPort:
     """SerialPort class with basic application-level protocol 
     functions to write strings and read strings"""
@@ -94,16 +97,16 @@ def writeStringExpectMatchingString(self, string, replyPattern):
         reply = self.readString()
         match = re.search(replyPattern, string)
         if match is None:
-            raise RuntimeError("No match")
+            raise CommunicationReadNoMatch("No match")
 
         return reply
 
     def writeStringReadFirstMatchingGroup(self, string, replyPattern):
         groups = self.writeStringReadMatchingGroups(string, replyPattern)
         if len(groups) >= 1:
-            return match.group(1)
+            return groups[0]
         else:
-            raise RuntimeError("No match")
+            raise CommunicationReadNoMatch("No match")
 
     def writeStringReadMatchingGroups(self, string, replyPattern):
         self.writeString(string)
@@ -113,7 +116,7 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
         if match is not None:
             return match.groups()
         else:
-            raise RuntimeError("No match")
+            raise CommunicationReadNoMatch("No match")
 
 
 class DebugEchoSerialPort(SerialPort):
diff --git a/testDebug.py b/testDebug.py
index 48ac88d5..98b6bc3a 100644
--- a/testDebug.py
+++ b/testDebug.py
@@ -77,6 +77,50 @@ def testTimeoutReadData(self):
             with self.assertRaises(CommunicationReadTimeout) as context:
                 self.port.readData(len(payloadData)+1)
 
+        def testWriteStringReadMatchingEcho(self):
+            reply = self.port.writeStringExpectMatchingString(
+                        payloadString,
+                        replyPattern=payloadString)
+            self.assertEqual(reply, payloadString)
+
+        def testWriteStringReadMatchingPattern(self):
+            reply = self.port.writeStringExpectMatchingString(
+                        "abcd1234\n",
+                        replyPattern="abc.\\d{4}")
+            self.assertEqual(reply, "abcd1234\n")
+
+        def testWriteStringReadMatchingPatternFirstCaptureGroup(self):
+            firstGroup = self.port.writeStringReadFirstMatchingGroup(
+                        "abcd1234\n",
+                        replyPattern="abc.(\\d{4})")
+            self.assertEqual(firstGroup, "1234")
+
+        def testWriteStringReadMatchingPatternAllCaptureGroups(self):
+            (firstGroup, secondGroup) = self.port.writeStringReadMatchingGroups(
+                        "abcd1234\n",
+                        replyPattern="(abc.)(\\d{4})")
+            self.assertEqual(firstGroup, "abcd")
+            self.assertEqual(secondGroup, "1234")
+
+        def testFailedWriteStringReadMatchingPattern(self):
+            with self.assertRaises(CommunicationReadNoMatch) as context:
+                reply = self.port.writeStringExpectMatchingString(
+                            "abcd1234\n",
+                            replyPattern="abc.\\d{5}")
+
+        def testFailedWriteStringReadMatchingPatternFirstCaptureGroup(self):
+            with self.assertRaises(CommunicationReadNoMatch) as context:
+                firstGroup = self.port.writeStringReadFirstMatchingGroup(
+                            "abcd1234\n",
+                            replyPattern="abc.(\\d{5})")
+
+
+        def testFailedWriteStringReadMatchingPatternAllCaptureGroups(self):
+            with self.assertRaises(CommunicationReadNoMatch) as context:
+                (firstGroup, secondGroup) = self.port.writeStringReadMatchingGroups(
+                            "abcd1234\n",
+                            replyPattern="(abc.)(\\d{5})")
+
 class TestDebugEchoPort(BaseTestCases.TestEchoPort):
 
     def setUp(self):
@@ -88,19 +132,19 @@ def tearDown(self):
         self.port.close()
 
 
-class TestRealEchoPort(BaseTestCases.TestEchoPort):
+# class TestRealEchoPort(BaseTestCases.TestEchoPort):
 
-    def setUp(self):
-        try:
-            self.port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
-            self.assertIsNotNone(self.port)
-            self.port.open()
-        except:
-            self.fail("Unable to setUp serial port")
+#     def setUp(self):
+#         try:
+#             self.port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
+#             self.assertIsNotNone(self.port)
+#             self.port.open()
+#         except:
+#             self.fail("Unable to setUp serial port")
 
 
-    def tearDown(self):
-        self.port.close()
+#     def tearDown(self):
+#         self.port.close()
 
 
 if __name__ == '__main__':

From 26fcd6910adf26b561daa217895ae4ba3394cf79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 17:32:41 -0400
Subject: [PATCH 11/17] Tested with real echo device, renamed files

---
 SerialPort.py               |  3 +++
 testDebug.py => testEcho.py | 20 ++++++++++----------
 tests.py                    | 20 --------------------
 3 files changed, 13 insertions(+), 30 deletions(-)
 rename testDebug.py => testEcho.py (93%)
 delete mode 100644 tests.py

diff --git a/SerialPort.py b/SerialPort.py
index 7a2fb1fd..792f8c9a 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -48,6 +48,9 @@ def readData(self, length):
         self.lock.acquire()
         try:
             data = self.port.read(length)
+            if len(data) != length:
+                raise CommunicationReadTimeout()
+
         finally:
             self.lock.acquire()
 
diff --git a/testDebug.py b/testEcho.py
similarity index 93%
rename from testDebug.py
rename to testEcho.py
index 98b6bc3a..2e53eb1f 100644
--- a/testDebug.py
+++ b/testEcho.py
@@ -132,19 +132,19 @@ def tearDown(self):
         self.port.close()
 
 
-# class TestRealEchoPort(BaseTestCases.TestEchoPort):
+class TestRealEchoPort(BaseTestCases.TestEchoPort):
 
-#     def setUp(self):
-#         try:
-#             self.port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
-#             self.assertIsNotNone(self.port)
-#             self.port.open()
-#         except:
-#             self.fail("Unable to setUp serial port")
+    def setUp(self):
+        try:
+            self.port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
+            self.assertIsNotNone(self.port)
+            self.port.open()
+        except:
+            self.fail("Unable to setUp serial port")
 
 
-#     def tearDown(self):
-#         self.port.close()
+    def tearDown(self):
+        self.port.close()
 
 
 if __name__ == '__main__':
diff --git a/tests.py b/tests.py
deleted file mode 100644
index 76307df6..00000000
--- a/tests.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import unittest
-from SerialPort import *
-
-
-class TestSerialPortWithEcho(unittest.TestCase):
-    serialPort = SerialPort("/dev/cu.usbserial-DPB3LCH1")
-
-    def setUp(self):
-        self.assertIsNotNone(self.serialPort)
-        self.serialPort.open()
-
-    def tearDown(self):
-        self.serialPort.close()
-
-    def testCreate(self):
-        self.assertIsNotNone(self.serialPort)
-
-
-if __name__ == '__main__':
-    unittest.main()

From 6cc6c886bad83e7adacff92ef7219df4c5fdf668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 17:33:31 -0400
Subject: [PATCH 12/17] NO need for all these modules

import errno
import fcntl
import os
import select
import struct
import sys
import termios
import array
import math
---
 SerialPort.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 792f8c9a..207d27fa 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -1,12 +1,3 @@
-import errno
-import fcntl
-import os
-import select
-import struct
-import sys
-import termios
-import array
-import math
 import serial
 import re
 from threading import Thread, RLock

From b2f6cad501de9e49a1515ddff5dc362ead71b7f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 17:49:42 -0400
Subject: [PATCH 13/17] Coverage nearly perfect

---
 SerialPort.py | 31 +++++++------------------------
 testEcho.py   | 11 +++++++++++
 2 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 207d27fa..def3f482 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -27,13 +27,7 @@ def close(self):
         self.port.close()
 
     def bytesAvailable(self):
-        return self.port.inwaiting()
-
-    def flush(self):
-        return None
-
-    def drain(self):
-        return None
+        return self.port.in_waiting
 
     def readData(self, length):
         self.lock.acquire()
@@ -53,6 +47,7 @@ def writeData(self, data):
             nBytesWritten = self.port.write(data)
             if nBytesWritten != len(data):
                 raise IOError("Not all bytes written to port")
+            self.port.flush()
         finally:
             self.lock.acquire()
 
@@ -100,7 +95,7 @@ def writeStringReadFirstMatchingGroup(self, string, replyPattern):
         if len(groups) >= 1:
             return groups[0]
         else:
-            raise CommunicationReadNoMatch("No match")
+            raise CommunicationReadNoMatch("Unable to find first group with pattern:'{0}'".format(replyPattern))
 
     def writeStringReadMatchingGroups(self, string, replyPattern):
         self.writeString(string)
@@ -110,7 +105,7 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
         if match is not None:
             return match.groups()
         else:
-            raise CommunicationReadNoMatch("No match")
+            raise CommunicationReadNoMatch("Unable to match pattern:'{0}' in reply:'{1}'".format(replyPattern, reply))
 
 
 class DebugEchoSerialPort(SerialPort):
@@ -124,6 +119,9 @@ def open(self):
     def close(self):
         return
 
+    def bytesAvailable(self):
+        return len(self.buffer)
+
     def readData(self, length):
         self.lock.acquire()
         try:
@@ -147,18 +145,3 @@ def writeData(self, data):
             self.lock.release()
 
         return len(data)
-
-
-if __name__ == "__main__":
-    port = DebugEchoSerialPort()
-    port.open()
-    port.close()
-
-    # port = SerialPort("/dev/cu.usbserial-ftDXIKC4")
-    # port.open()
-    # port.writeString("abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd\n")
-    # print(port.readString())
-    # print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcd"))
-    # print(port.writeStringExpectMatchingString(string="abcd\n", replyPattern="abcad"))
-    # print(port.writeStringReadFirstMatchingGroup(string="abcd\n", replyPattern="ab(cd)"))
-    # port.close()
diff --git a/testEcho.py b/testEcho.py
index 2e53eb1f..c77be431 100644
--- a/testEcho.py
+++ b/testEcho.py
@@ -1,6 +1,7 @@
 import unittest
 from serial import *
 from SerialPort import *
+import time
 
 payloadData = b'1234'
 payloadString = '1234\n'
@@ -17,6 +18,11 @@ def testWriteData(self):
             nBytes = self.port.writeData(payloadData)
             self.assertTrue(nBytes == len(payloadData))
 
+        def testWriteDataBytesAvailable(self):
+            nBytes = self.port.writeData(payloadData)
+            time.sleep(1)
+            self.assertEqual(nBytes, self.port.bytesAvailable())
+
         def testWriteDataReadEcho(self):
             nBytes = self.port.writeData(payloadData)
             self.assertTrue(nBytes == len(payloadData))
@@ -114,6 +120,11 @@ def testFailedWriteStringReadMatchingPatternFirstCaptureGroup(self):
                             "abcd1234\n",
                             replyPattern="abc.(\\d{5})")
 
+        def testFailedWriteStringReadMatchingPatternWithoutGroupFirstCaptureGroup(self):
+            with self.assertRaises(CommunicationReadNoMatch) as context:
+                firstGroup = self.port.writeStringReadFirstMatchingGroup(
+                            "abcd1234\n",
+                            replyPattern="abc.\\d{4}")
 
         def testFailedWriteStringReadMatchingPatternAllCaptureGroups(self):
             with self.assertRaises(CommunicationReadNoMatch) as context:

From 3eb9ab2702267194cdf2c0a519c9c938e2dc2b2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 21:50:13 -0400
Subject: [PATCH 14/17] Added thread safety tests

but testWriteDataReadEcho() fails sometimes
---
 SerialPort.py | 55 ++++++++++++++++++++++++++++++++-------------------
 testEcho.py   | 45 +++++++++++++++++++++++++++++++++++++----
 2 files changed, 76 insertions(+), 24 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index def3f482..7ca6c6c0 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -37,7 +37,7 @@ def readData(self, length):
                 raise CommunicationReadTimeout()
 
         finally:
-            self.lock.acquire()
+            self.lock.release()
 
         return data
 
@@ -49,7 +49,7 @@ def writeData(self, data):
                 raise IOError("Not all bytes written to port")
             self.port.flush()
         finally:
-            self.lock.acquire()
+            self.lock.release()
 
         return nBytesWritten
 
@@ -82,30 +82,45 @@ def writeString(self, string):
         return nBytes
 
     def writeStringExpectMatchingString(self, string, replyPattern):
-        self.writeString(string)
-        reply = self.readString()
-        match = re.search(replyPattern, string)
-        if match is None:
-            raise CommunicationReadNoMatch("No match")
+        self.lock.acquire()
+
+        try:
+            self.writeString(string)
+            reply = self.readString()
+            match = re.search(replyPattern, string)
+            if match is None:
+                raise CommunicationReadNoMatch("No match")
+        finally:
+            self.lock.release()
 
         return reply
 
     def writeStringReadFirstMatchingGroup(self, string, replyPattern):
-        groups = self.writeStringReadMatchingGroups(string, replyPattern)
-        if len(groups) >= 1:
-            return groups[0]
-        else:
-            raise CommunicationReadNoMatch("Unable to find first group with pattern:'{0}'".format(replyPattern))
+        self.lock.acquire()
+
+        try:
+            groups = self.writeStringReadMatchingGroups(string, replyPattern)
+            if len(groups) >= 1:
+                return groups[0]
+            else:
+                raise CommunicationReadNoMatch("Unable to find first group with pattern:'{0}'".format(replyPattern))
+        finally:
+            self.lock.release()
 
     def writeStringReadMatchingGroups(self, string, replyPattern):
-        self.writeString(string)
-        reply = self.readString()
-        match = re.search(replyPattern, string)
-
-        if match is not None:
-            return match.groups()
-        else:
-            raise CommunicationReadNoMatch("Unable to match pattern:'{0}' in reply:'{1}'".format(replyPattern, reply))
+        self.lock.acquire()
+
+        try:
+            self.writeString(string)
+            reply = self.readString()
+            match = re.search(replyPattern, string)
+
+            if match is not None:
+                return match.groups()
+            else:
+                raise CommunicationReadNoMatch("Unable to match pattern:'{0}' in reply:'{1}'".format(replyPattern, reply))
+        finally:
+            self.lock.release()
 
 
 class DebugEchoSerialPort(SerialPort):
diff --git a/testEcho.py b/testEcho.py
index c77be431..d7cc6ff1 100644
--- a/testEcho.py
+++ b/testEcho.py
@@ -2,9 +2,12 @@
 from serial import *
 from SerialPort import *
 import time
+from threading import Thread, Lock
 
 payloadData = b'1234'
 payloadString = '1234\n'
+globalLock = Lock()
+threadFailed = -1
 
 class BaseTestCases:
 
@@ -18,10 +21,9 @@ def testWriteData(self):
             nBytes = self.port.writeData(payloadData)
             self.assertTrue(nBytes == len(payloadData))
 
-        def testWriteDataBytesAvailable(self):
-            nBytes = self.port.writeData(payloadData)
-            time.sleep(1)
-            self.assertEqual(nBytes, self.port.bytesAvailable())
+        # def testWriteDataBytesAvailable(self):
+        #     nBytes = self.port.writeData(payloadData)
+        #     self.assertEqual(nBytes, self.port.bytesAvailable())
 
         def testWriteDataReadEcho(self):
             nBytes = self.port.writeData(payloadData)
@@ -132,6 +134,41 @@ def testFailedWriteStringReadMatchingPatternAllCaptureGroups(self):
                             "abcd1234\n",
                             replyPattern="(abc.)(\\d{5})")
 
+        def testThreadSafety(self):
+            global threadFailed
+            threadFailed = -1
+            threadPool = []
+            for i in range(100):
+                process = Thread(target=threadReadWrite, kwargs=dict(port=self.port, index=i))
+                process.start()
+                threadPool.append(process)
+
+            for i,t in enumerate(threadPool):
+                t.join(timeout=2)
+
+                try:
+                    globalLock.acquire()
+                    if threadFailed != -1:
+                        self.fail("Thread {0}?".format(threadFailed))
+                finally:
+                    globalLock.release()
+
+        
+def threadReadWrite(port, index):
+    global threadFailed
+    payload = "abcd{0}\n".format(index)
+    globalLock.acquire()
+    try:
+        port.writeStringExpectMatchingString(payload, replyPattern=payload)
+    except:
+        if threadFailed != -1:
+            threadFailed = index        
+    finally:
+        globalLock.release()
+
+
+
+
 class TestDebugEchoPort(BaseTestCases.TestEchoPort):
 
     def setUp(self):

From 7560ef2b9cfb39707a6727b2f17ef9160dc87141 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 21:55:47 -0400
Subject: [PATCH 15/17] new lock: transaction and port locks

transaction is a write/read group.
portLock is access to the port.
---
 SerialPort.py | 42 ++++++++++++++++++++++--------------------
 testEcho.py   |  4 +++-
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 7ca6c6c0..e21ce544 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -18,10 +18,12 @@ def __init__(self, bsdPath=None, vendorId=None,
         self.vendorId = vendorId
         self.productId = productId
         self.serialNumber = serialNumber
-        self.lock = RLock()
+        self.portLock = RLock()
+        self.transactionLock = RLock()
+
 
     def open(self):
-        self.port = serial.Serial(self.bsdPath, 19200, timeout=1)
+        self.port = serial.Serial(self.bsdPath, 19200, timeout=0.3)
 
     def close(self):
         self.port.close()
@@ -30,31 +32,31 @@ def bytesAvailable(self):
         return self.port.in_waiting
 
     def readData(self, length):
-        self.lock.acquire()
+        self.portLock.acquire()
         try:
             data = self.port.read(length)
             if len(data) != length:
                 raise CommunicationReadTimeout()
 
         finally:
-            self.lock.release()
+            self.portLock.release()
 
         return data
 
     def writeData(self, data):
-        self.lock.acquire()
+        self.portLock.acquire()
         try:
             nBytesWritten = self.port.write(data)
             if nBytesWritten != len(data):
                 raise IOError("Not all bytes written to port")
             self.port.flush()
         finally:
-            self.lock.release()
+            self.portLock.release()
 
         return nBytesWritten
 
     def readString(self):
-        self.lock.acquire()
+        self.portLock.acquire()
         try:
             byte = None
             data = bytearray(0)
@@ -66,23 +68,23 @@ def readString(self):
 
             string = data.decode(encoding='utf-8')
         finally:
-            self.lock.release()
+            self.portLock.release()
 
         return string
 
     def writeString(self, string):
-        self.lock.acquire()
+        self.portLock.acquire()
         nBytes = 0
         try:
             data = bytearray(string, "utf-8")
             nBytes = self.writeData(data)
         finally:
-            self.lock.release()
+            self.portLock.release()
 
         return nBytes
 
     def writeStringExpectMatchingString(self, string, replyPattern):
-        self.lock.acquire()
+        self.transactionLock.acquire()
 
         try:
             self.writeString(string)
@@ -91,12 +93,12 @@ def writeStringExpectMatchingString(self, string, replyPattern):
             if match is None:
                 raise CommunicationReadNoMatch("No match")
         finally:
-            self.lock.release()
+            self.transactionLock.release()
 
         return reply
 
     def writeStringReadFirstMatchingGroup(self, string, replyPattern):
-        self.lock.acquire()
+        self.transactionLock.acquire()
 
         try:
             groups = self.writeStringReadMatchingGroups(string, replyPattern)
@@ -105,10 +107,10 @@ def writeStringReadFirstMatchingGroup(self, string, replyPattern):
             else:
                 raise CommunicationReadNoMatch("Unable to find first group with pattern:'{0}'".format(replyPattern))
         finally:
-            self.lock.release()
+            self.transactionLock.release()
 
     def writeStringReadMatchingGroups(self, string, replyPattern):
-        self.lock.acquire()
+        self.transactionLock.acquire()
 
         try:
             self.writeString(string)
@@ -120,7 +122,7 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
             else:
                 raise CommunicationReadNoMatch("Unable to match pattern:'{0}' in reply:'{1}'".format(replyPattern, reply))
         finally:
-            self.lock.release()
+            self.transactionLock.release()
 
 
 class DebugEchoSerialPort(SerialPort):
@@ -138,7 +140,7 @@ def bytesAvailable(self):
         return len(self.buffer)
 
     def readData(self, length):
-        self.lock.acquire()
+        self.portLock.acquire()
         try:
             data = bytearray()
             for i in range(0, length):
@@ -148,15 +150,15 @@ def readData(self, length):
                 else:
                     raise CommunicationReadTimeout("Unable to read data")
         finally:
-            self.lock.release()
+            self.portLock.release()
 
         return data
 
     def writeData(self, data):
-        self.lock.acquire()
+        self.portLock.acquire()
         try:
             self.buffer.extend(data)
         finally:
-            self.lock.release()
+            self.portLock.release()
 
         return len(data)
diff --git a/testEcho.py b/testEcho.py
index d7cc6ff1..d1033a74 100644
--- a/testEcho.py
+++ b/testEcho.py
@@ -140,9 +140,11 @@ def testThreadSafety(self):
             threadPool = []
             for i in range(100):
                 process = Thread(target=threadReadWrite, kwargs=dict(port=self.port, index=i))
-                process.start()
                 threadPool.append(process)
 
+            for process in threadPool:
+                process.start()
+
             for i,t in enumerate(threadPool):
                 t.join(timeout=2)
 

From 710a9719c9e3b16a948c55ab62169786b4a6fbb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 22:09:55 -0400
Subject: [PATCH 16/17] Better lock syntax

self.portLock.acquire()
        try:
        finally:
             self.port.release()

replaced by
        with self.portLock:
---
 SerialPort.py | 41 ++++++++---------------------------------
 testEcho.py   | 11 +++++++----
 2 files changed, 15 insertions(+), 37 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index e21ce544..7670dbaf 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -32,32 +32,24 @@ def bytesAvailable(self):
         return self.port.in_waiting
 
     def readData(self, length):
-        self.portLock.acquire()
-        try:
+        with self.portLock:
             data = self.port.read(length)
             if len(data) != length:
                 raise CommunicationReadTimeout()
 
-        finally:
-            self.portLock.release()
-
         return data
 
     def writeData(self, data):
-        self.portLock.acquire()
-        try:
+        with self.portLock:
             nBytesWritten = self.port.write(data)
             if nBytesWritten != len(data):
                 raise IOError("Not all bytes written to port")
             self.port.flush()
-        finally:
-            self.portLock.release()
 
         return nBytesWritten
 
-    def readString(self):
-        self.portLock.acquire()
-        try:
+    def readString(self):      
+        with self.portLock:
             byte = None
             data = bytearray(0)
             while (byte != b''):
@@ -67,52 +59,37 @@ def readString(self):
                     break
 
             string = data.decode(encoding='utf-8')
-        finally:
-            self.portLock.release()
 
         return string
 
     def writeString(self, string):
-        self.portLock.acquire()
         nBytes = 0
-        try:
+        with self.portLock:
             data = bytearray(string, "utf-8")
             nBytes = self.writeData(data)
-        finally:
-            self.portLock.release()
 
         return nBytes
 
     def writeStringExpectMatchingString(self, string, replyPattern):
-        self.transactionLock.acquire()
-
-        try:
+        with self.transactionLock:
             self.writeString(string)
             reply = self.readString()
             match = re.search(replyPattern, string)
             if match is None:
                 raise CommunicationReadNoMatch("No match")
-        finally:
-            self.transactionLock.release()
 
         return reply
 
     def writeStringReadFirstMatchingGroup(self, string, replyPattern):
-        self.transactionLock.acquire()
-
-        try:
+        with self.transactionLock:
             groups = self.writeStringReadMatchingGroups(string, replyPattern)
             if len(groups) >= 1:
                 return groups[0]
             else:
                 raise CommunicationReadNoMatch("Unable to find first group with pattern:'{0}'".format(replyPattern))
-        finally:
-            self.transactionLock.release()
 
     def writeStringReadMatchingGroups(self, string, replyPattern):
-        self.transactionLock.acquire()
-
-        try:
+        with self.transactionLock:
             self.writeString(string)
             reply = self.readString()
             match = re.search(replyPattern, string)
@@ -121,8 +98,6 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
                 return match.groups()
             else:
                 raise CommunicationReadNoMatch("Unable to match pattern:'{0}' in reply:'{1}'".format(replyPattern, reply))
-        finally:
-            self.transactionLock.release()
 
 
 class DebugEchoSerialPort(SerialPort):
diff --git a/testEcho.py b/testEcho.py
index d1033a74..223d5802 100644
--- a/testEcho.py
+++ b/testEcho.py
@@ -135,7 +135,7 @@ def testFailedWriteStringReadMatchingPatternAllCaptureGroups(self):
                             replyPattern="(abc.)(\\d{5})")
 
         def testThreadSafety(self):
-            global threadFailed
+            global threadFailed, globalLock
             threadFailed = -1
             threadPool = []
             for i in range(100):
@@ -145,8 +145,8 @@ def testThreadSafety(self):
             for process in threadPool:
                 process.start()
 
-            for i,t in enumerate(threadPool):
-                t.join(timeout=2)
+            for i,p in enumerate(threadPool):
+                p.join(timeout=1)
 
                 try:
                     globalLock.acquire()
@@ -155,9 +155,12 @@ def testThreadSafety(self):
                 finally:
                     globalLock.release()
 
+            for i,p in enumerate(threadPool):
+                p.join(timeout=1)
+
         
 def threadReadWrite(port, index):
-    global threadFailed
+    global threadFailed, globalLock
     payload = "abcd{0}\n".format(index)
     globalLock.acquire()
     try:

From 89c640c57b38206f28a598bae54f3b897aa09d8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20C=C3=B4t=C3=A9?= <dccote@novajo.ca>
Date: Sun, 19 May 2019 22:17:57 -0400
Subject: [PATCH 17/17] Added a Slow port for testing with a delay property in
 debugSerrialPort

---
 SerialPort.py | 14 ++++++--------
 testEcho.py   | 27 ++++++++++++++++++---------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/SerialPort.py b/SerialPort.py
index 7670dbaf..8c376b3f 100644
--- a/SerialPort.py
+++ b/SerialPort.py
@@ -1,6 +1,8 @@
 import serial
 import re
 from threading import Thread, RLock
+import time
+import random
 
 class CommunicationReadTimeout(serial.SerialException):
     pass
@@ -103,6 +105,7 @@ def writeStringReadMatchingGroups(self, string, replyPattern):
 class DebugEchoSerialPort(SerialPort):
     def __init__(self):
         self.buffer = bytearray()
+        self.delay = 0
         super(DebugEchoSerialPort, self).__init__()
 
     def open(self):
@@ -115,8 +118,8 @@ def bytesAvailable(self):
         return len(self.buffer)
 
     def readData(self, length):
-        self.portLock.acquire()
-        try:
+        with self.portLock:
+            time.sleep(self.delay*random.random())
             data = bytearray()
             for i in range(0, length):
                 if len(self.buffer) > 0:
@@ -124,16 +127,11 @@ def readData(self, length):
                     data.append(byte)
                 else:
                     raise CommunicationReadTimeout("Unable to read data")
-        finally:
-            self.portLock.release()
 
         return data
 
     def writeData(self, data):
-        self.portLock.acquire()
-        try:
+        with self.portLock:
             self.buffer.extend(data)
-        finally:
-            self.portLock.release()
 
         return len(data)
diff --git a/testEcho.py b/testEcho.py
index 223d5802..69674456 100644
--- a/testEcho.py
+++ b/testEcho.py
@@ -162,23 +162,32 @@ def testThreadSafety(self):
 def threadReadWrite(port, index):
     global threadFailed, globalLock
     payload = "abcd{0}\n".format(index)
-    globalLock.acquire()
-    try:
-        port.writeStringExpectMatchingString(payload, replyPattern=payload)
-    except:
-        if threadFailed != -1:
-            threadFailed = index        
-    finally:
-        globalLock.release()
+    with globalLock:
+        try:
+            port.writeStringExpectMatchingString(payload, replyPattern=payload)
+        except:
+            if threadFailed != -1:
+                threadFailed = index        
 
 
+class TestDebugEchoPort(BaseTestCases.TestEchoPort):
 
+    def setUp(self):
+        self.port = DebugEchoSerialPort()
+        self.assertIsNotNone(self.port)
+        self.port.open()
 
-class TestDebugEchoPort(BaseTestCases.TestEchoPort):
+    def tearDown(self):
+        self.port.close()
+
+
+
+class TestSlowDebugEchoPort(BaseTestCases.TestEchoPort):
 
     def setUp(self):
         self.port = DebugEchoSerialPort()
         self.assertIsNotNone(self.port)
+        self.port.delay = 0.01
         self.port.open()
 
     def tearDown(self):