From 80432d82cf81b5d84c257964c9ce63adf5d38833 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 18 Sep 2023 20:15:33 +1000 Subject: [PATCH] Market Data Enhancements #7 Proof-of-concept of UDP/Multicast re-implementation. This will require re-work to handle the case of multiple simultanenous OrderBooks, each order book will then have to broadcast over Kafka and then a listener pushes out the stream and adds sequence numbers. The future state model will also have to support A, B and C (...Z) UDP/Multicast streams. The sequence number implementation suggests a single thread somewhere to ensure each of the A, B, C...Z streams uses the same sequence number for the same logical message. --- administrator/version.properties | 4 +- .../alignmentsystems/matching/OrderBook.java | 5 +- .../matching/OrderBookKafkaProducer.java | 19 +++- .../matching/OrderBookWrapper.java | 23 ++++- .../matching/udp/MulticastServer.java | 91 +++++++------------ exchange/version.properties | 4 +- fixclient/version.properties | 4 +- .../interfaces/InterfaceMulticastServer.java | 16 +--- .../interfaces/InterfaceOrderBook.java | 2 - .../interfaces/InterfaceOrderBookEvents.java | 7 ++ sharedlibrary/version.properties | 4 +- 11 files changed, 90 insertions(+), 89 deletions(-) create mode 100644 sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBookEvents.java diff --git a/administrator/version.properties b/administrator/version.properties index d0a6b54..beb46ef 100644 --- a/administrator/version.properties +++ b/administrator/version.properties @@ -1,2 +1,2 @@ -#Mon Sep 18 19:15:13 AEST 2023 -VERSION_BUILD=704 +#Mon Sep 18 20:08:14 AEST 2023 +VERSION_BUILD=711 diff --git a/exchange/src/main/java/com/alignmentsystems/matching/OrderBook.java b/exchange/src/main/java/com/alignmentsystems/matching/OrderBook.java index 5915b4b..c1e0c06 100644 --- a/exchange/src/main/java/com/alignmentsystems/matching/OrderBook.java +++ b/exchange/src/main/java/com/alignmentsystems/matching/OrderBook.java @@ -37,6 +37,7 @@ import com.alignmentsystems.library.interfaces.InterfaceMatchEvent; import com.alignmentsystems.library.interfaces.InterfaceOrder; import com.alignmentsystems.library.interfaces.InterfaceOrderBook; +import com.alignmentsystems.library.interfaces.InterfaceOrderBookEvents; import quickfix.FieldNotFound; @@ -47,7 +48,7 @@ * */ public class OrderBook -implements KafkaMessageHandler, InterfaceOrderBook, InterfaceMatchEvent, InterfaceAddedOrderToOrderBook, Runnable { +implements KafkaMessageHandler, InterfaceOrderBook, InterfaceMatchEvent, InterfaceAddedOrderToOrderBook, InterfaceOrderBookEvents, Runnable { private final static String CLASSNAME = OrderBook.class.getSimpleName(); private final static int buyPriorityQueueSize = 100; @@ -57,9 +58,9 @@ public class OrderBook private PriorityQueue buy = new PriorityQueue(buyPriorityQueueSize, aboc); private PriorityQueue sell = new PriorityQueue(sellPriorityQueueSize, asoc); - private List listenersMatchEvent = new ArrayList(); private PersistenceToFileClient debugger = null; + private List listenersMatchEvent = new ArrayList(); private List listenersAddedOrderToOrderBook = new ArrayList(); private AtomicBoolean running = new AtomicBoolean(false); private AtomicBoolean initialised = new AtomicBoolean(false); diff --git a/exchange/src/main/java/com/alignmentsystems/matching/OrderBookKafkaProducer.java b/exchange/src/main/java/com/alignmentsystems/matching/OrderBookKafkaProducer.java index 95af1be..445fcf1 100644 --- a/exchange/src/main/java/com/alignmentsystems/matching/OrderBookKafkaProducer.java +++ b/exchange/src/main/java/com/alignmentsystems/matching/OrderBookKafkaProducer.java @@ -11,6 +11,8 @@ *****************************************************************************/ import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; @@ -26,15 +28,25 @@ import com.alignmentsystems.library.interfaces.InterfaceExecutionReport; import com.alignmentsystems.library.interfaces.InterfaceMatch; import com.alignmentsystems.library.interfaces.InterfaceMatchEvent; +import com.alignmentsystems.library.interfaces.InterfaceOrderBookEvents; -public class OrderBookKafkaProducer implements InterfaceMatchEvent, InterfaceAddedOrderToOrderBook, Runnable { +public class OrderBookKafkaProducer implements InterfaceMatchEvent, InterfaceAddedOrderToOrderBook, InterfaceOrderBookEvents, Runnable { private KafkaProducer kafkaProducerB = null; private LogEncapsulation log = null; private static BinaryFromToCanonical binaryFromToCanonical = new BinaryFromToCanonical(); + private List listenersMatchEvent = new ArrayList(); + private List listenersAddedOrderToOrderBook = new ArrayList(); - - + @Override + public void addAddedOrderToOrderBookListener(InterfaceAddedOrderToOrderBook toAdd) { + this.listenersAddedOrderToOrderBook.add(toAdd); + } + + @Override + public void addMatchEventListener(InterfaceMatchEvent toAdd) { + this.listenersMatchEvent.add(toAdd); + } public OrderBookKafkaProducer() { // TODO Auto-generated constructor stub @@ -119,4 +131,5 @@ public void addedOrderToOrderBook(InterfaceExecutionReport er) { byte[] ack = binaryFromToCanonical.getBufferFromExecutionReport(er); this.send(TOPIC, er.getExecID().toString(), ack); } + } diff --git a/exchange/src/main/java/com/alignmentsystems/matching/OrderBookWrapper.java b/exchange/src/main/java/com/alignmentsystems/matching/OrderBookWrapper.java index 08875fa..d15277c 100644 --- a/exchange/src/main/java/com/alignmentsystems/matching/OrderBookWrapper.java +++ b/exchange/src/main/java/com/alignmentsystems/matching/OrderBookWrapper.java @@ -21,18 +21,20 @@ import com.alignmentsystems.library.interfaces.InterfaceMatchEvent; import com.alignmentsystems.library.interfaces.InterfaceOrderBook; import com.alignmentsystems.library.interfaces.InterfaceOrderBookWrapper; +import com.alignmentsystems.matching.udp.MulticastServer; /** * @author John Greenan * */ public class OrderBookWrapper implements InterfaceOrderBookWrapper{ + @SuppressWarnings("unused") private final static String CLASSNAME = OrderBookWrapper.class.getSimpleName(); private Map orderBooks = new HashMap(); private LogEncapsulation log = null; private PersistenceToFileClient debugger = null; - private List listenersMatchEvent = new ArrayList(); + //private List listenersMatchEvent = new ArrayList(); // private List listenersAddedOrderToOrderBook = // new ArrayList(); @@ -56,7 +58,7 @@ public boolean initialise(LogEncapsulation log, PersistenceToFileClient debugger OrderBook orderBook = new OrderBook(); - OrderBookKafkaProducer obkp = null; + OrderBookKafkaProducer obkp = new OrderBookKafkaProducer(); obkp.initialise(log); @@ -65,6 +67,18 @@ public boolean initialise(LogEncapsulation log, PersistenceToFileClient debugger }else{ return false; }; + + MulticastServer multicast = new MulticastServer(); + + + Thread multiCastServerThread = new Thread(null , multicast, MulticastServer.CLASSNAME); + + multiCastServerThread.start(); + + + obkp.addAddedOrderToOrderBookListener(multicast); + obkp.addMatchEventListener(multicast); + Thread obThread = new Thread(orderBook); @@ -82,6 +96,11 @@ public boolean initialise(LogEncapsulation log, PersistenceToFileClient debugger throw e; } + //orderBook.addAddedOrderToOrderBookListener(multicast); + //orderBook.addMatchEventListener(multicast); + + + try { obkc.runAlways(symbol, orderBook); diff --git a/exchange/src/main/java/com/alignmentsystems/matching/udp/MulticastServer.java b/exchange/src/main/java/com/alignmentsystems/matching/udp/MulticastServer.java index ba149f4..3383b12 100644 --- a/exchange/src/main/java/com/alignmentsystems/matching/udp/MulticastServer.java +++ b/exchange/src/main/java/com/alignmentsystems/matching/udp/MulticastServer.java @@ -31,19 +31,21 @@ import com.alignmentsystems.library.constants.Constants; import com.alignmentsystems.library.enumerations.ConfigurationProperties; import com.alignmentsystems.library.enumerations.InstanceType; +import com.alignmentsystems.library.interfaces.InterfaceAddedOrderToOrderBook; +import com.alignmentsystems.library.interfaces.InterfaceExecutionReport; import com.alignmentsystems.library.interfaces.InterfaceMatch; +import com.alignmentsystems.library.interfaces.InterfaceMatchEvent; import com.alignmentsystems.library.interfaces.InterfaceMulticastServer; /** * @author John Greenan * */ -public class MulticastServer implements Runnable, InterfaceMulticastServer { - private final static String CLASSNAME = MulticastServer.class.getSimpleName().toString(); +public class MulticastServer implements InterfaceMulticastServer , InterfaceMatchEvent, InterfaceAddedOrderToOrderBook, Runnable{ + public final static String CLASSNAME = MulticastServer.class.getSimpleName().toString(); private DatagramSocket socket; private InetAddress group; private LogEncapsulation log = null; - private ConcurrentLinkedQueue marketDataToPublishQueue = null; private PersistenceToFileClient debugger = null; private String host = null; private int port = 0; @@ -99,10 +101,9 @@ private boolean innerInitialise() { } @Override - public boolean initialise(LogEncapsulation log, ConcurrentLinkedQueue marketDataToPublishQueue, - PersistenceToFileClient debugger, String host, int port) { + public boolean initialise(LogEncapsulation log, PersistenceToFileClient debugger, String host, int port) { this.log = log; - this.marketDataToPublishQueue = marketDataToPublishQueue; + this.debugger = debugger; this.host = host; this.port = port; @@ -111,54 +112,10 @@ public boolean initialise(LogEncapsulation log, ConcurrentLinkedQueue " + payload); - - this.multicastThis(bb.array()); - - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - try { - Thread.currentThread(); - Thread.sleep(MILLISLEEP); - - } catch (InterruptedException e) { - - running.set(false); - - Thread.currentThread().interrupt(); - - System.err.println(e.getMessage()); - - System.err.println(new StringBuilder().append(CLASSNAME).append(Constants.SPACE).append(e.getMessage()) - .toString()); - } - } - } - - @Override - public boolean initialise(LogEncapsulation log, ConcurrentLinkedQueue marketDataToPublishQueue, - PersistenceToFileClient debugger) throws Exception{ + public boolean initialise( + LogEncapsulation log + , PersistenceToFileClient debugger) throws Exception{ this.log = log; - this.marketDataToPublishQueue = marketDataToPublishQueue; this.debugger = debugger; try { this.host = LibraryFunctions.getProperty(MulticastServer.class.getClassLoader() , InstanceType.MULTICASTSERVER.getProperties(), ConfigurationProperties.MULTICASTHOST); @@ -167,18 +124,36 @@ public boolean initialise(LogEncapsulation log, ConcurrentLinkedQueue getMarketDataQueue() { - return this.marketDataToPublishQueue; + public void matchHappened(InterfaceMatch arg0) { + // TODO Auto-generated method stub + } @Override - public Long getCurrentSequenceNumber() { - return ++sequenceNumber; + public void run() { + AtomicBoolean run = new AtomicBoolean(true); + while (run.get()) { + try { + wait(2000); + } catch (InterruptedException e) { + log.error(e.getMessage() , e ); + } + } + } } \ No newline at end of file diff --git a/exchange/version.properties b/exchange/version.properties index d1ef3eb..640448c 100644 --- a/exchange/version.properties +++ b/exchange/version.properties @@ -1,2 +1,2 @@ -#Mon Sep 18 19:15:14 AEST 2023 -VERSION_BUILD=702 +#Mon Sep 18 20:08:14 AEST 2023 +VERSION_BUILD=709 diff --git a/fixclient/version.properties b/fixclient/version.properties index 74aaf97..beb46ef 100644 --- a/fixclient/version.properties +++ b/fixclient/version.properties @@ -1,2 +1,2 @@ -#Mon Sep 18 19:15:14 AEST 2023 -VERSION_BUILD=704 +#Mon Sep 18 20:08:14 AEST 2023 +VERSION_BUILD=711 diff --git a/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceMulticastServer.java b/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceMulticastServer.java index 0b3062b..8b90729 100644 --- a/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceMulticastServer.java +++ b/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceMulticastServer.java @@ -1,16 +1,5 @@ package com.alignmentsystems.library.interfaces; import java.io.IOException; -/****************************************************************************** - * - * Author : John Greenan - * Contact : sales@alignment-systems.com - * Date : 24th August 2023 - * Copyright : Alignment Systems Ltd 2023 - * Project : Alignment Matching Toy - * Artefact : InterfaceMulticastServer - * Description : - *****************************************************************************/ -import java.util.concurrent.ConcurrentLinkedQueue; import com.alignmentsystems.library.LogEncapsulation; import com.alignmentsystems.library.PersistenceToFileClient; @@ -19,9 +8,8 @@ * */ public interface InterfaceMulticastServer { - public boolean initialise(LogEncapsulation log, ConcurrentLinkedQueue marketDataToPublishQueue, PersistenceToFileClient debugger, String host, int port) throws Exception; - public boolean initialise(LogEncapsulation log, ConcurrentLinkedQueue marketDataToPublishQueue, PersistenceToFileClient debugger) throws Exception; + public boolean initialise(LogEncapsulation log, PersistenceToFileClient debugger, String host, int port) throws Exception; + public boolean initialise(LogEncapsulation log, PersistenceToFileClient debugger) throws Exception; public void multicastThis(byte[] multicastBytes) throws IOException; - public ConcurrentLinkedQueue getMarketDataQueue(); public Long getCurrentSequenceNumber(); } \ No newline at end of file diff --git a/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBook.java b/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBook.java index 12f11dd..6341731 100644 --- a/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBook.java +++ b/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBook.java @@ -34,8 +34,6 @@ public Boolean initialise( public OffsetDateTime getOrderBookCreationTime(); public OffsetDateTime getOrderBookLastUpdateTime(); public int getOrderCountBySide(OrderBookSide orderBookSide ); - public void addMatchEventListener(InterfaceMatchEvent toAdd); - public void addAddedOrderToOrderBookListener(InterfaceAddedOrderToOrderBook toAdd); public void upsertTopOfBook(InterfaceOrder nos); public void updateLevelsOfDepth(); } \ No newline at end of file diff --git a/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBookEvents.java b/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBookEvents.java new file mode 100644 index 0000000..24e16c3 --- /dev/null +++ b/sharedlibrary/src/main/java/com/alignmentsystems/library/interfaces/InterfaceOrderBookEvents.java @@ -0,0 +1,7 @@ +package com.alignmentsystems.library.interfaces; + +public interface InterfaceOrderBookEvents { + public void addMatchEventListener(InterfaceMatchEvent toAdd); + public void addAddedOrderToOrderBookListener(InterfaceAddedOrderToOrderBook toAdd); + +} diff --git a/sharedlibrary/version.properties b/sharedlibrary/version.properties index d1ef3eb..640448c 100644 --- a/sharedlibrary/version.properties +++ b/sharedlibrary/version.properties @@ -1,2 +1,2 @@ -#Mon Sep 18 19:15:14 AEST 2023 -VERSION_BUILD=702 +#Mon Sep 18 20:08:14 AEST 2023 +VERSION_BUILD=709