java_api.clj

(ns libpython-clj2.java-api
  "A java api is exposed for libpython-clj2.  The methods below are statically callable
  without the leading '-'.  Note that returned python objects implement the respective
  java interfaces so a python dict will implement java.util.Map, etc.  There is some
  startup time as Clojure dynamically compiles the source code but this binding should
  have great runtime characteristics in comparison to any other java python engine.


  Note that you can pass java objects into python.  An implementation of java.util.Map will
  appear to python as a dict-like object an implementation of java.util.List will look
  like a sequence and if the thing is iterable then it will be iterable in python.
  To receive callbacks from python you can provide an implementation of the
  interface [clojure.lang.IFn](https://clojure.github.io/clojure/javadoc/clojure/lang/IFn.html)
  - see [clojure.lang.AFn](https://github.com/clojure/clojure/blob/master/src/jvm/clojure/lang/AFn.java) for a base class that makes this easier.


  Performance:

  There are a two logical ways to repeatedly invoke python functionality.  The first is
  to set some globals and repeatedly [[runStringAsInput]] a script.  The second is
  'exec'ing a script, finding an exported function in the global dict and calling
  that.  With libpython-clj, the second pathway -- repeatedly calling a function --
  is going to be faster than the first if the user makes a fastcallable out of the function
  to be invoked.  Here are some sample timings for an extremely simple function with two
  arguments:


```console
Python fn calls/ms 1923.2490094806021
Python fastcallable calls/ms 3776.767751742239
Python eval pathway calls/ms 2646.0478013509883
```


  * [[makeFastcallable]] - - For the use case of repeatedly calling a single function - this
  will cache the argument tuple for repeated use as opposed to allocating the argument tuple
  every call.  This can be a surprising amount faster -- 2x-3x -- than directly calling the
  python callable.  Once a fastcallable object is made you can either cast it to a
  [clojure.lang.IFn](https://clojure.github.io/clojure/javadoc/clojure/lang/IFn.html)
  or call it via the provided `call` static method.


  * HAIR ON FIRE MODE - If you are certain you are correctly calling lockGIL and unlockGIL
  then you can define a variable, `-Dlibpython_clj.manual_gil=true` that will disable
  automatic GIL lock/unlock system and gil correctness checking.  This is useful, for instance,
  if you are going to lock libpython-clj to a thread and control all access to it yourself.
  This pathway will get at most 10% above using fastcall by itself.


  Example:


```java
  java_api.initialize(null);
  try (AutoCloseable locker = java_api.GILLocker()) {
    np = java_api.importModule(\"numpy\");
    Object ones = java_api.getAttr(np, \"ones\");
    ArrayList dims = new ArrayList();
    dims.add(2);
    dims.add(3);
    Object npArray = java_api.call(ones, dims); //see fastcall notes above
    ...
  }
```"
  (:import [java.util Map Map$Entry List HashMap LinkedHashMap Set HashSet]
           [java.util.function Supplier]
           [java.lang AutoCloseable]
           [clojure.java.api Clojure]
           [com.google.common.cache CacheBuilder RemovalListener])
  (:gen-class
   :name libpython_clj2.java_api
   :main false
   :methods [#^{:static true} [initialize [java.util.Map] Object]
             #^{:static true} [initializeEmbedded [] Object]
             #^{:static true} [lockGIL [] long]
             #^{:static true} [unlockGIL [long] Object]
             #^{:static true} [GILLocker [] java.lang.AutoCloseable]
             #^{:static true} [hasAttr [Object String] Boolean]
             #^{:static true} [getAttr [Object String] Object]
             #^{:static true} [setAttr [Object String Object] Object]
             #^{:static true} [hasItem [Object Object] Boolean]
             #^{:static true} [getItem [Object Object] Object]
             #^{:static true} [setItem [Object Object Object] Object]
             #^{:static true} [getGlobal [String] Object]
             #^{:static true} [setGlobal [String Object] Object]
             #^{:static true} [runStringAsInput [String] Object]
             #^{:static true} [runStringAsFile [String] java.util.Map]
             #^{:static true} [importModule [String] Object]
             #^{:static true} [callKw [Object java.util.List java.util.Map] Object]
             #^{:static true} [call [Object] Object]
             #^{:static true} [call [Object Object] Object]
             #^{:static true} [call [Object Object Object] Object]
             #^{:static true} [call [Object Object Object Object] Object]
             #^{:static true} [call [Object Object Object Object Object] Object]
             #^{:static true} [call [Object Object Object Object Object Object] Object]
             #^{:static true} [call [Object Object Object Object Object Object Object] Object]
             #^{:static true} [allocateFastcallContext [] Object]
             #^{:static true} [releaseFastcallContext [Object] Object]
             ;;args require context
             #^{:static true} [fastcall [Object Object] Object]
             #^{:static true} [fastcall [Object Object Object] Object] ;;1
             #^{:static true} [fastcall [Object Object Object Object] Object] ;;2
             #^{:static true} [fastcall [Object Object Object Object Object] Object] ;;3
             #^{:static true} [fastcall [Object Object Object Object Object Object] Object] ;;4
             #^{:static true} [makeFastcallable [Object] java.lang.AutoCloseable]
             #^{:static true} [copyToPy [Object] Object]
             #^{:static true} [copyToJVM [Object] Object]
             #^{:static true} [createArray [String Object Object] Object]
             #^{:static true} [arrayToJVM [Object] java.util.Map]
             #^{:static true} [copyData [Object Object] Object]]
   ))


(set! *warn-on-reflection* true)


(def ^:private requires* (delay
                           (let [require (Clojure/var "clojure.core" "require")]
                             (doseq [clj-ns ["tech.v3.datatype"
                                             "tech.v3.tensor"
                                             "libpython-clj2.python"
                                             "libpython-clj2.python.fn"
                                             "libpython-clj2.python.ffi"
                                             "libpython-clj2.python.gc"
                                             "libpython-clj2.python.np-array"]]
                               (require (Clojure/read clj-ns))))))


(def ^:private ->python* (delay (Clojure/var "libpython-clj2.python" "->python")))
(def ^:private fastcall* (delay (Clojure/var "libpython-clj2.python.fn" "fastcall")))
(def ^:private allocate-fastcall-context* (delay (Clojure/var "libpython-clj2.python.fn" "allocate-fastcall-context")))
(def ^:private release-fastcall-context* (delay (Clojure/var "libpython-clj2.python.fn" "release-fastcall-context")))
(def ^:private make-fastcallable* (delay (Clojure/var "libpython-clj2.python.fn" "make-fastcallable")))
(def ^:private check-error-throw* (delay (Clojure/var "libpython-clj2.python.ffi" "check-error-throw")))
(def ^:private simplify-or-track* (delay (Clojure/var "libpython-clj2.python.ffi" "simplify-or-track")))
(def ^:private as-jvm* (delay (Clojure/var "libpython-clj2.python" "as-jvm")))
(def ^:private ->jvm* (delay (Clojure/var "libpython-clj2.python" "->jvm")))
(def ^:private eval-code* (delay (Clojure/var "libpython-clj2.python.ffi" "PyEval_EvalCode")))
(def ^:private untracked->python* (delay (Clojure/var "libpython-clj2.python.ffi" "untracked->python")))
(def ^:private decref* (delay (Clojure/var "libpython-clj2.python.ffi" "Py_DecRef")))


(defn- decref-lru-cache
  ^Map [^long max-size]
  (let [builder (CacheBuilder/newBuilder)]
    (.maximumSize builder max-size)
    (.removalListener builder (reify RemovalListener
                                (onRemoval [this notification]
                                  (@decref* (.getValue notification)))))
    (-> (.build builder)
        (.asMap))))


(def ^{:tag Map :private true} string-cache (decref-lru-cache 1024))
(declare -lockGIL -unlockGIL)
(def ^:private globals*
  (delay
    (let [gstate (-lockGIL)
          main-mod ((Clojure/var "libpython-clj2.python.ffi" "PyImport_AddModule") "__main__")]
      (-> ((Clojure/var "libpython-clj2.python.ffi" "PyModule_GetDict") main-mod)
          (@as-jvm*)))))

(def ^:private strcomp* (delay (let [->python @->python*]
                                 (reify java.util.function.Function
                                   (apply [this data]
                                     (->python data))))))


(declare -createArray)


(def ^{:tag 'Map
       :private true}
  primitive-arrays
  (let [base-arrays [[(byte-array 0) {:length #(alength ^bytes %)
                                      :datatype "int8"}]
                     [(short-array 0) {:length #(alength ^shorts %)
                                       :datatype "int16"}]
                     [(int-array 0) {:length #(alength ^ints %)
                                     :datatype "int32"}]
                     [(long-array 0) {:length #(alength ^longs %)
                                      :datatype "int64"}]
                     [(float-array 0) {:length #(alength ^floats %)
                                       :datatype "float32"}]
                     [(double-array 0) {:length #(alength ^doubles %)
                                        :datatype "float64"}]]
        typeset (HashMap.)]
    (doseq [[ary getter] base-arrays]
      (.put typeset (type ary) getter))
    typeset))


(def ^{:tag 'Set
       :private true}
  array-types
  (let [base-arrays [(byte-array 0)
                     (short-array 0)
                     (int-array 0)
                     (long-array 0)
                     (float-array 0)
                     (double-array 0)]
        array-of-arrays (->> base-arrays
                             (map (fn [ary]
                                    (into-array [ary]))))
        typeset (HashSet.)]

    (doseq [t array-of-arrays]
      (.add typeset (type t)))
    (.addAll typeset (.keySet primitive-arrays))
    typeset))


#_(defn- to-python
  "Support for auto-converting primitive arrays and array-of-arrays into python."
  [item]
  (let [item-type (type item)]
    (if (.contains array-types item-type)
      (if-let [len-getter (get primitive-arrays item-type)]
        (-createArray (len-getter :datatype)
                      [((len-getter :length) item)]
                      item)
        ;;potential 2d array if all lengths match
        (let [^objects item item
              item-len (alength item)
              [datatype shape]
              (when-let [fitem (when-not (== 0 item-len)
                                 (aget item 0))]
                (let [ary-entry (get primitive-arrays (type fitem))
                      inner-len (get ary-entry :length)
                      ary-dt (get ary-entry :datatype)
                      fitem-len (unchecked-long (inner-len fitem))
                      matching?
                      (loop [idx 1
                             matching? true]
                        (if (< idx item-len)
                          (recur (unchecked-inc idx)
                                 (and matching?
                                      (== fitem-len
                                          (unchecked-long
                                           (if-let [inner-item (aget item idx)]
                                             (inner-len inner-item)
                                             0)))))
                          matching?))]
                  (when matching?
                    [ary-dt [item-len fitem-len]])))]
          (if shape
            (-createArray datatype shape item)
            ;;slower fallback for ragged arrays
            (@->python* item))))
      (@->python* item))))


(def ^:private fast-dict-set-item*
  (delay (let [untracked->python (Clojure/var "libpython-clj2.python.ffi"
                                              "untracked->python")
               ->python @->python*
               decref (Clojure/var "libpython-clj2.python.ffi"
                                   "Py_DecRef")
               pydict-setitem (Clojure/var "libpython-clj2.python.ffi"
                                           "PyDict_SetItem")]
           (fn [dict k v]
             (let [v (untracked->python v ->python)]
               (pydict-setitem dict k v)
               (decref v))))))


(defn- cached-string
  [strval]
  (when strval
    (.computeIfAbsent string-cache strval @strcomp*)))


(def ^{:tag Map :private true} compile-cache (decref-lru-cache 1024))

(def ^:private compiler*
  (delay
    (let [compile-fn (Clojure/var "libpython-clj2.python.ffi"
                                  "Py_CompileString")]
      (reify java.util.function.Function
        (apply [this [strdata run-type]]
          (compile-fn strdata "unnamed.py" (case run-type
                                             :file 257
                                             :input 258)))))))

(defn- compile-string
  [strval run-type]
  (when strval
    (.computeIfAbsent compile-cache [strval run-type] @compiler*)))


(defn -initialize
  "See options for [[libpython-clj2.python/initialize!]].  Note that the keyword
  option arguments can be provided by using strings so `:library-path` becomes
  the key \"library-path\".  Also note that there is still a GIL underlying
  all of the further operations so java should access python via single-threaded
  pathways."
  [options]
  @requires*
  (apply (Clojure/var "libpython-clj2.python" "initialize!")
         (->> options
              (mapcat (fn [^Map$Entry entry]
                        [(keyword (.getKey entry)) (.getValue entry)])))))


(defn -initializeEmbedded
  "Initialize python when this library is being called *from* a python program.  In
  that case the system will look for the python symbols in the current executable.
  See the [embedded topic](https://clj-python.github.io/libpython-clj/embedded.html)"
  []
  @requires*
  ((Clojure/var "libpython-clj2.python.ffi" "set-library!") nil))


(defn -lockGIL
  "Attempt to lock the gil.  This is safe to call in a reentrant manner.
  Returns a long representing the gil state that must be passed into unlockGIL.

  See documentation for [pyGILState_Ensure](https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure).

  Note that the API will do this for you but locking
  the GIL before doing a string of operations is faster than having each operation lock
  the GIL individually."
  []
  ((Clojure/var "libpython-clj2.python.ffi" "lock-gil")))


(defn -unlockGIL
  "Unlock the gil passing in the gilstate returned from lockGIL.  Each call to lockGIL must
  be paired to a call to unlockGIL."
  [gilstate]
  ((Clojure/var "libpython-clj2.python.ffi" "unlock-gil") gilstate))


(defn -GILLocker
  "Lock the gil returning an AutoCloseable that will unlock the gil upon close.

  Example:

```java
  try (AutoCloseable locker = java_api.GILLocker()) {
    ... Do your python stuff.
  }
```"
  ^AutoCloseable []
  (let [gilstate (-lockGIL)]
    (reify AutoCloseable
      (close [this]
        (-unlockGIL gilstate)))))


(defn -hasAttr
  "Returns true if this python object has this attribute."
  [pyobj attname]
  (boolean ((Clojure/var "libpython-clj2.python" "has-attr?") pyobj attname)))


(defn -getAttr
  "Get a python attribute.  This corresponds to the python '.' operator."
  [pyobj attname]
  ((Clojure/var "libpython-clj2.python" "get-attr") pyobj attname))


(defn -setAttr
  "Set an attribute on a python object.  This corresponds to the `__setattr` python call."
  [pyobj attname objval]
  ((Clojure/var "libpython-clj2.python" "set-attr!") pyobj attname objval))


(defn -hasItem
  "Return true if this pyobj has this item"
  [pyobj itemName]
  (boolean ((Clojure/var "libpython-clj2.python" "has-item?") pyobj itemName)))


(defn -getItem
  [pyobj itemName]
  ((Clojure/var "libpython-clj2.python" "get-item") pyobj itemName))


(defn -setItem
  [pyobj itemName itemVal]
  ((Clojure/var "libpython-clj2.python" "set-item!") pyobj itemName itemVal))


(defn -setGlobal
  "Set a value in the global dict.  This function expects the GIL to be locked - it will
  not lock/unlock it for you.

  In addition to numbers and strings, this method can take an implementation of
  `clojure.lang.IFn` that will be converted to a python callable, arrays and lists
  will be converted to python lists.  If you would like numpy arrays use [[-createArray]]."
  [^String varname varval]
  (@fast-dict-set-item* @globals* (cached-string varname) varval)
  nil)


(defn -getGlobal
  "Get a value from the global dict.  This function expects the GIL to be locked - it will
  not lock/unlock it for you."
  [^String varname]
  (-getItem @globals* (cached-string varname)))


(defn -runStringAsInput
  "Run a string returning the result of the last expression.  Strings are compiled and
  live for the life of the interpreter.  This is the equivalent to the python
  [eval](https://docs.python.org/3/library/functions.html#eval) call.

  This function expects the GIL to be locked - it will not lock/unlock it for you.

  Example:

```java
  java_api.setGlobal(\"bid\", 1);
  java_api.setGlobal(\"ask\", 2);
  java_api.runStringAsInput(\"bid-ask\"); //Returns -1
```"
  [strdata]
  (let [pyobj (compile-string strdata :input)]
    (when-not pyobj
      (@check-error-throw*))
    (if-let [script-rt (@eval-code* pyobj @globals* @globals*)]
      (-> (@simplify-or-track* script-rt)
          (@as-jvm*)))))


(defn -runStringAsFile
  "Run a string returning the result of the last expression.  Strings are compiled and
  live for the life of the interpreter.  This is the equivalent to the python
  [exec](https://docs.python.org/3/library/functions.html#exec) all.

  The global context is returned as a java map.

  This function expects the GIL to be locked - it will not lock/unlock it for you.

Example:

```java
   Map globals = java_api.runStringAsFile(\"def calcSpread(bid,ask):\n\treturn bid-ask\n\n\");
   Object spreadFn = globals.get(\"calcSpread\");
   java_api.call(spreadFn, 1, 2); // Returns -1
```"
  [strdata]
  (let [pyobj (compile-string strdata :file)]
    (when-not pyobj
      (@check-error-throw*))
    (@eval-code* pyobj @globals* @globals*)
    (@check-error-throw*)
    @globals*))


(defn -importModule
  "Import a python module.  Module entries can be accessed via `getAttr`."
  [modname]
  ((Clojure/var "libpython-clj2.python" "import-module") modname))


(defn -callKw
  "Call a python callable with keyword arguments.  Note that you don't need this pathway
  to call python methods if you do not need keyword arguments; if the python object is
  callable then it will implement [clojure.lang.IFn](https://clojure.github.io/clojure/javadoc/clojure/lang/IFn.html) and you can use `invoke`."
  [pyobj pos-args kw-args]
  (let [gstate (-lockGIL)]
    (try
      ((Clojure/var "libpython-clj2.python.fn" "call-kw")
       pyobj pos-args (->> kw-args
                           (map (fn [^Map$Entry entry]
                                  [(.getKey entry)
                                   (.getValue entry)]))
                           (into {})))
      (finally
        (-unlockGIL gstate)))))


(defn -call
  "Call a clojure `IFn` object.  Python callables implement this interface so this works for
  python objects.  This is a convenience method around casting implementation of
  [clojure.lang.IFn](https://clojure.github.io/clojure/javadoc/clojure/lang/IFn.html) and
  calling `invoke` directly."
  ([item]
   (item))
  ([item arg1]
   (item arg1))
  ([item arg1 arg2]
   (item arg1 arg2))
  ([item arg1 arg2 arg3]
   (item arg1 arg2 arg3))
  ([item arg1 arg2 arg3 arg4]
   (item arg1 arg2 arg3 arg4))
  ([item arg1 arg2 arg3 arg4 arg5]
   (item item arg1 arg2 arg3 arg4 arg5))
  ([item arg1 arg2 arg3 arg4 arg5 arg6]
   (item item arg1 arg2 arg3 arg4 arg5 arg6)))


(defn ^:no-doc -allocateFastcallContext
  "Allocate a fastcall context.  See docs for [[-fastcall]].  The returned context must be
  release via [[-releaseFastcallContext]]."
  []
  (@allocate-fastcall-context*))


(defn ^:no-doc -releaseFastcallContext
  "Release a fastcall context.  See docs for [[-fastcall]].  This is safe to call with
  null and to call multiple times on the same context."
  [ctx]
  (when ctx
    (@release-fastcall-context* ctx)))


(defn ^:no-doc -fastcall
  "Call a python function as fast as possible using a fixed number of positional arguments.
  The caller must provide an allocated fastcall context
  via [[-allocateFastcallContext]] and furthermore the caller may choose to deallocate the
  fastcall context with [[-releaseFastcallContext]].

  Current overloads support arities up to 4 arguments.  You must not use the same context
  with different arities - contexts are allocated in an arity-specific manner.

  For a more condensed version see [[makeFastcallable]]."
  ([ctx item]
   ;;ctx is unused but placed here to allow rapid search/replace to be correct.
   (@fastcall* item))
  ([ctx item arg1]
   (@fastcall* ctx item arg1))
  ([ctx item arg1 arg2]
   (@fastcall* ctx item arg1 arg2))
  ([ctx item arg1 arg2 arg3]
   (@fastcall* ctx item arg1 arg2 arg3))
  ([ctx item arg1 arg2 arg3 arg4]
   (@fastcall* ctx item arg1 arg2 arg3 arg4)))


(defn -makeFastcallable
  "Given a normal python callable, make a fastcallable object that needs to be closed.
  This should be seen as a callsite optimization for repeatedly calling a specific python
  function in a tight loop with positional arguments.  It is not intended to be used in a
  context where you will then pass this object around as this is not a reentrant optimization.

```java
  try (AutoCloseable fastcaller = java_api.makeFastcallable(pycallable)) {
     tightloop: {
       java_api.call(fastcaller, arg1, arg2);
     }
  }
```"
  ^java.lang.AutoCloseable [item]
  (@make-fastcallable* item))


(defn -copyToPy
  "Copy a basic jvm object, such as an implementation of java.util.Map or java.util.List to
  a python object."
  [object]
  ((Clojure/var "libpython-clj2.python" "->python") object))


(defn -copyToJVM
  "Copy a python object such as a dict or a list into a comparable JVM object."
  [object]
  ((Clojure/var "libpython-clj2.python" "->jvm") object))


(defn -createArray
  "Create a numpy array from a tuple of string datatype, shape and data.
  * `datatype` - One of \"int8\" \"uint8\" \"int16\" \"uint16\" \"int32\" \"uint32\"
  \"int64\" \"uint64\" \"float32\" \"float64\".
  * `shape` - integer array of dimension e.g. `[2,3]`.
  * `data` - list or array of data.  This will of course be fastest if the datatype
  of the array matches the requested datatype.

  This does work with array-of-array structures assuming the shape is correct but those
  will be slower than a single primitive array and a shape."
  [datatype shape data]
  (let [reshape (Clojure/var "tech.v3.tensor" "reshape")
        ->python @->python*]
    (-> ((Clojure/var "tech.v3.tensor" "->tensor") data :datatype (keyword datatype))
        (reshape shape)
        (->python))))


(defn -arrayToJVM
  "Copy (efficiently) a numeric numpy array into a jvm map containing keys \"datatype\",
  \"shape\", and a jvm array \"data\" in flattened row-major form."
  [pyobj]
  (let [tens-data ((Clojure/var "libpython-clj2.python" "as-jvm") pyobj)]
    {"datatype" (name ((Clojure/var "tech.v3.datatype" "elemwise-datatype") tens-data))
     "shape" (int-array ((Clojure/var "tech.v3.datatype" "shape") tens-data))
     "data" ((Clojure/var "tech.v3.datatype" "->array") tens-data)}))


(def ^:private copyfn* (delay (Clojure/var "tech.v3.datatype" "copy!")))


(defn -copyData
  "Copy data from a jvm container into a numpy array or back.  This allows you to use a fixed
  preallocated set of numpy (and potentially jvm arrays) to transfer data back and forth
  efficiently.  The most efficient transfer will be from a java primitive array that matches
  the numeric type of the numpy array.  Also note the element count of the numpy array
  and the jvm array must match.

  Note this copies *from* the first argument *to* the second argument -- this is reverse the
  normal memcpy argument order!!.  Returns the destination (to) argument.

  Not this does *not* work with array-of-arrays.  It will work with, for instance,
  a numpy matrix of shape [2, 2] and an double array of length 4."
  [from to]
  (@copyfn* from to))