(capi) - Reusability, cosmetics and performance improvements

Author: protocolstardust

Makefile

@@ -27,7 +27,7 @@ pull_rayforce_from_github:
 patch_rayforce_makefile:
 	@echo "🔧 Patching Makefile for Python support..."
 	@echo '\n# Build Python module' >> $(EXEC_DIR)/tmp/rayforce-c/Makefile
-	@echo 'PY_OBJECTS = core/rayforce_c.o core/raypy_conversion.o core/raypy_constructors.o core/raypy_readers.o core/raypy_operations.o core/raypy_queries.o core/raypy_io.o' >> $(EXEC_DIR)/tmp/rayforce-c/Makefile
+	@echo 'PY_OBJECTS = core/rayforce_c.o core/raypy_init_from_py.o core/raypy_read_from_rf.o core/raypy_queries.o core/raypy_io.o core/raypy_binary.o core/raypy_dynlib.o core/raypy_eval.o core/raypy_iter.o' >> $(EXEC_DIR)/tmp/rayforce-c/Makefile
 	@echo 'PY_APP_OBJECTS = app/term.o' >> $(EXEC_DIR)/tmp/rayforce-c/Makefile
 	@echo 'python: CFLAGS = $$(RELEASE_CFLAGS) -I$$(shell python3 -c "import sysconfig; print(sysconfig.get_config_var(\"INCLUDEPY\"))") -Wno-macro-redefined' >> $(EXEC_DIR)/tmp/rayforce-c/Makefile
 	@echo 'python: LDFLAGS = $(RELEASE_LDFLAGS)' >> $(EXEC_DIR)/tmp/rayforce-c/Makefile
@@ -53,12 +53,14 @@ clean:
 rayforce_binaries:
 	@cp rayforce/capi/rayforce_c.c tmp/rayforce-c/core/rayforce_c.c
 	@cp rayforce/capi/rayforce_c.h tmp/rayforce-c/core/rayforce_c.h
-	@cp rayforce/capi/raypy_conversion.c tmp/rayforce-c/core/raypy_conversion.c
-	@cp rayforce/capi/raypy_constructors.c tmp/rayforce-c/core/raypy_constructors.c
-	@cp rayforce/capi/raypy_readers.c tmp/rayforce-c/core/raypy_readers.c
-	@cp rayforce/capi/raypy_operations.c tmp/rayforce-c/core/raypy_operations.c
+	@cp rayforce/capi/raypy_init_from_py.c tmp/rayforce-c/core/raypy_init_from_py.c
+	@cp rayforce/capi/raypy_read_from_rf.c tmp/rayforce-c/core/raypy_read_from_rf.c
 	@cp rayforce/capi/raypy_queries.c tmp/rayforce-c/core/raypy_queries.c
 	@cp rayforce/capi/raypy_io.c tmp/rayforce-c/core/raypy_io.c
+	@cp rayforce/capi/raypy_binary.c tmp/rayforce-c/core/raypy_binary.c
+	@cp rayforce/capi/raypy_dynlib.c tmp/rayforce-c/core/raypy_dynlib.c
+	@cp rayforce/capi/raypy_eval.c tmp/rayforce-c/core/raypy_eval.c
+	@cp rayforce/capi/raypy_iter.c tmp/rayforce-c/core/raypy_iter.c
 	@cd tmp/rayforce-c && $(MAKE) python
 	@cd tmp/rayforce-c && $(MAKE) release
 	@cd tmp/rayforce-c/ext/raykx && $(MAKE) release
@@ -81,6 +83,7 @@ lint:
 	python3 -m ruff format tests/ rayforce/
 	python3 -m ruff check rayforce/ --fix
 	python3 -m mypy rayforce/
+	clang-format -i rayforce/capi/*
 
 ipython:
 	ipython -i -c "from rayforce import *"

rayforce/_rayforce_c.pyi

@@ -1,3 +1,4 @@
+from collections.abc import Sequence
 from typing import Any
 
 TYPE_LIST: int
@@ -49,10 +50,10 @@ def init_date(value: Any) -> RayObject: ...
 def init_time(value: Any) -> RayObject: ...
 def init_timestamp(value: Any) -> RayObject: ...
 def init_guid(value: Any) -> RayObject: ...
-def init_list() -> RayObject: ...
+def init_list(item: list[Any]) -> RayObject: ...
 def init_table(columns: RayObject, values: RayObject) -> RayObject: ...
 def init_dict(keys: RayObject, values: RayObject) -> RayObject: ...
-def init_vector(type_code: int, length: int) -> RayObject: ...
+def init_vector(type_code: int, length_or_items: int | Sequence[Any]) -> RayObject: ...
 def read_i16(obj: RayObject) -> int: ...
 def read_i32(obj: RayObject) -> int: ...
 def read_i64(obj: RayObject) -> int: ...
@@ -76,20 +77,17 @@ def at_idx(iterable: RayObject, idx: int) -> RayObject: ...
 def insert_obj(iterable: RayObject, idx: int, ptr: RayObject) -> None: ...
 def push_obj(iterable: RayObject, ptr: RayObject) -> None: ...
 def set_obj(obj: RayObject, idx: RayObject, value: RayObject) -> None: ...
-def fill_vector(obj: RayObject, fill: list[Any]) -> None: ...
-def fill_list(obj: RayObject, fill: list[Any]) -> None: ...
 def get_obj_length(obj: RayObject) -> int: ...
 def eval_str(obj: RayObject) -> RayObject: ...
 def get_error_obj(error_obj: RayObject) -> RayObject: ...
 def binary_set(name: RayObject, obj: RayObject) -> None: ...
-def env_get_internal_function_by_name(name: str) -> RayObject: ...
-def env_get_internal_name_by_function(obj: RayObject) -> str: ...
+def env_get_internal_fn_by_name(name: str) -> RayObject: ...
+def env_get_internal_name_by_fn(obj: RayObject) -> str: ...
 def eval_obj(obj: RayObject) -> RayObject: ...
 def loadfn_from_file(filename: str, fn_name: str, args_count: int) -> RayObject: ...
 def quote(obj: RayObject) -> RayObject: ...
 def rc_obj(obj: RayObject) -> int: ...
 def set_obj_attrs(obj: RayObject, attr: int) -> None: ...
-def select(query: RayObject) -> RayObject: ...
 def update(query: RayObject) -> RayObject: ...
 def insert(table: RayObject, data: RayObject) -> RayObject: ...
 def upsert(table: RayObject, keys: RayObject, data: RayObject) -> RayObject: ...

rayforce/capi/rayforce_c.c

@@ -10,10 +10,7 @@ int check_main_thread(void) {
     return 0;
   }
 
-  unsigned long current_thread_id;
-  current_thread_id = (unsigned long)PyThread_get_thread_ident();
-
-  if (current_thread_id != g_main_thread_id) {
+  if ((unsigned long)PyThread_get_thread_ident() != g_main_thread_id) {
     PyErr_Format(PyExc_RuntimeError,
                  "Rayforce runtime can not be called from other threads than "
                  "the one where it was initialized from");
@@ -23,58 +20,58 @@ int check_main_thread(void) {
   return 1;
 }
 
-// Initialize runtime (FFI function)
+// GENERIC UTILS
 PyObject *raypy_init_runtime(PyObject *self, PyObject *args) {
   (void)self;
-  (void)args; // Suppress unused parameter warning
+  (void)args;
 
   if (g_runtime != NULL) {
-    // Runtime already initialized
-    Py_RETURN_NONE;
+    PyErr_SetString(PyExc_RuntimeError, "Runtime already initialized");
+    return NULL;
   }
 
-  char *argv[] = {"raypy", "-r", "0", NULL};
-  g_runtime = runtime_create(3, argv);
-  if (g_runtime == NULL) {
+  char *argv[] = {"py", "-r", "0", NULL};
+  if (runtime_create(3, argv) == NULL) {
     PyErr_SetString(PyExc_RuntimeError, "Failed to initialize Rayforce");
     return NULL;
   }
 
-  if (g_main_thread_id == 0)
-    g_main_thread_id = (unsigned long)PyThread_get_thread_ident();
-
+  g_main_thread_id = (unsigned long)PyThread_get_thread_ident();
   Py_RETURN_NONE;
 }
+PyObject *raypy_wrap_ray_object(obj_p ray_obj) {
+  if (ray_obj == NULL) {
+    PyErr_SetString(PyExc_RuntimeError, "Rayforce object can't be null");
+    return NULL;
+  }
+
+  RayObject *result = (RayObject *)RayObjectType.tp_alloc(&RayObjectType, 0);
+  if (result != NULL)
+    result->obj = ray_obj;
+  return (PyObject *)result;
+}
+// --
+
+static void RayObject_dealloc(RayObject *self) {
+  if (self->obj != NULL && self->obj != NULL_OBJ)
+    drop_obj(self->obj);
+  Py_TYPE(self)->tp_free((PyObject *)self);
+}
 
 PyTypeObject RayObjectType = {
     PyVarObject_HEAD_INIT(NULL, 0).tp_name = "_rayforce_c.RayObject",
     .tp_basicsize = sizeof(RayObject),
     .tp_itemsize = 0,
-    .tp_dealloc = NULL, // Will be set below
+    .tp_dealloc = (destructor)RayObject_dealloc,
     .tp_flags = Py_TPFLAGS_DEFAULT,
     .tp_doc = "RayObject objects",
-    .tp_methods = NULL, // Will be set below
+    .tp_methods = NULL,
     .tp_new = PyType_GenericNew,
 };
 
-static void RayObject_dealloc(RayObject *self) {
-  if (self->obj != NULL && self->obj != NULL_OBJ)
-    drop_obj(self->obj);
-  Py_TYPE(self)->tp_free((PyObject *)self);
-}
-
 PyMODINIT_FUNC PyInit__rayforce_c(void);
 
-static struct PyModuleDef rayforce_module = {
-    PyModuleDef_HEAD_INIT,
-    .m_name = "_rayforce_c",
-    .m_doc = "Python C API bus to Rayforce",
-    .m_size = -1,
-    .m_methods = NULL, // Will be set in PyInit__rayforce_c
-};
-
-static PyMethodDef module_methods[] = {
-    // Constructors
+static PyMethodDef rayforce_methods[] = {
     {"init_i16", raypy_init_i16, METH_VARARGS, "Create a new i16 object"},
     {"init_i32", raypy_init_i32, METH_VARARGS, "Create a new i32 object"},
     {"init_i64", raypy_init_i64, METH_VARARGS, "Create a new i64 object"},
@@ -99,8 +96,6 @@ static PyMethodDef module_methods[] = {
      "Create a new dictionary object"},
     {"init_vector", raypy_init_vector, METH_VARARGS,
      "Create a new vector object"},
-
-    // Readers
     {"read_i16", raypy_read_i16, METH_VARARGS, "Read i16 value from object"},
     {"read_i32", raypy_read_i32, METH_VARARGS, "Read i32 value from object"},
     {"read_i64", raypy_read_i64, METH_VARARGS, "Read i64 value from object"},
@@ -117,40 +112,26 @@ static PyMethodDef module_methods[] = {
     {"read_timestamp", raypy_read_timestamp, METH_VARARGS,
      "Read timestamp value from object"},
     {"read_guid", raypy_read_guid, METH_VARARGS, "Read GUID value from object"},
-
-    // Table operations
     {"table_keys", raypy_table_keys, METH_VARARGS, "Get table keys"},
     {"table_values", raypy_table_values, METH_VARARGS, "Get table values"},
     {"repr_table", raypy_repr_table, METH_VARARGS, "Format table"},
-
-    // Dictionary operations
     {"dict_keys", raypy_dict_keys, METH_VARARGS, "Get dictionary keys"},
     {"dict_values", raypy_dict_values, METH_VARARGS, "Get dictionary values"},
     {"dict_get", raypy_dict_get, METH_VARARGS, "Get value from dictionary"},
-
-    // Vector operations
     {"at_idx", raypy_at_idx, METH_VARARGS, "Get element at index"},
     {"insert_obj", raypy_insert_obj, METH_VARARGS, "Insert object at index"},
     {"push_obj", raypy_push_obj, METH_VARARGS,
      "Push object to the end of iterable"},
     {"set_obj", raypy_set_obj, METH_VARARGS, "Set object at index"},
-    {"fill_vector", raypy_fill_vector, METH_VARARGS,
-     "Fill vector from Python list (bulk operation)"},
-    {"fill_list", raypy_fill_list, METH_VARARGS,
-     "Fill list from Python list (bulk operation)"},
-
-    // Misc operations
     {"get_obj_length", raypy_get_obj_length, METH_VARARGS, "Get object length"},
     {"eval_str", raypy_eval_str, METH_VARARGS, "Evaluate string expression"},
     {"get_error_obj", raypy_get_error_obj, METH_VARARGS, "Get error object"},
     {"binary_set", raypy_binary_set, METH_VARARGS,
      "Set value to symbol or file"},
-    {"env_get_internal_function_by_name",
-     raypy_env_get_internal_function_by_name, METH_VARARGS,
-     "Get internal function by name"},
-    {"env_get_internal_name_by_function",
-     raypy_env_get_internal_name_by_function, METH_VARARGS,
-     "Get internal function name"},
+    {"env_get_internal_fn_by_name", raypy_env_get_internal_fn_by_name,
+     METH_VARARGS, "Get internal function by name"},
+    {"env_get_internal_name_by_fn", raypy_env_get_internal_name_by_fn,
+     METH_VARARGS, "Get internal function name"},
     {"eval_obj", raypy_eval_obj, METH_VARARGS, "Evaluate object"},
     {"loadfn_from_file", raypy_loadfn, METH_VARARGS,
      "Load function from shared library"},
@@ -159,35 +140,30 @@ static PyMethodDef module_methods[] = {
     {"set_obj_attrs", raypy_set_obj_attrs, METH_VARARGS,
      "Set object attributes"},
     {"get_obj_type", raypy_get_obj_type, METH_VARARGS, "Get object type"},
-
-    // Database operations
-    {"select", raypy_select, METH_VARARGS, "Perform SELECT query"},
     {"update", raypy_update, METH_VARARGS, "Perform UPDATE query"},
     {"insert", raypy_insert, METH_VARARGS, "Perform INSERT query"},
     {"upsert", raypy_upsert, METH_VARARGS, "Perform UPSERT query"},
-
-    // IO operations
     {"hopen", raypy_hopen, METH_VARARGS, "Open file or socket handle"},
     {"hclose", raypy_hclose, METH_VARARGS, "Close file or socket handle"},
     {"write", raypy_write, METH_VARARGS, "Write data to file or socket"},
-
     {"init_runtime", raypy_init_runtime, METH_VARARGS,
      "Initialize Rayforce runtime"},
 
     {NULL, NULL, 0, NULL}};
 
+static struct PyModuleDef rayforce_module = {
+    PyModuleDef_HEAD_INIT, .m_name = "_rayforce_c",
+    .m_doc = "Python C API bus to Rayforce", .m_size = -1,
+    .m_methods = rayforce_methods};
+
 static RayObject *g_null_obj = NULL;
 
 PyMODINIT_FUNC PyInit__rayforce_c(void) {
   PyObject *m;
 
-  // Initialize RayObjectType
-  RayObjectType.tp_dealloc = (destructor)RayObject_dealloc;
-
   if (PyType_Ready(&RayObjectType) < 0)
     return NULL;
 
-  rayforce_module.m_methods = module_methods;
   m = PyModule_Create(&rayforce_module);
   if (m == NULL)
     return NULL;

rayforce/capi/rayforce_c.h

@@ -51,7 +51,6 @@ typedef struct {
   PyObject_HEAD obj_p obj;
 } RayObject;
 
-// Runtime
 extern void *g_runtime;
 
 int check_main_thread(void);
@@ -62,7 +61,12 @@ int check_main_thread(void);
       return NULL;                                                             \
   } while (0)
 
-// Conversion functions (Python -> Rayforce)
+#if defined(__GNUC__) || defined(__clang__)
+#define UNUSED_SELF_PARAM __attribute__((unused))
+#else
+#define UNUSED_SELF_PARAM
+#endif
+
 obj_p raypy_init_i16_from_py(PyObject *item);
 obj_p raypy_init_i32_from_py(PyObject *item);
 obj_p raypy_init_i64_from_py(PyObject *item);
@@ -71,14 +75,29 @@ obj_p raypy_init_c8_from_py(PyObject *item);
 obj_p raypy_init_b8_from_py(PyObject *item);
 obj_p raypy_init_u8_from_py(PyObject *item);
 obj_p raypy_init_symbol_from_py(PyObject *item);
+obj_p raypy_init_string_from_py(PyObject *item);
+obj_p raypy_init_list_from_py(PyObject *item);
 obj_p raypy_init_guid_from_py(PyObject *item);
 obj_p raypy_init_date_from_py(PyObject *item);
 obj_p raypy_init_time_from_py(PyObject *item);
 obj_p raypy_init_timestamp_from_py(PyObject *item);
 obj_p raypy_init_dict_from_py(PyObject *item);
 obj_p raypy_init_list_from_py(PyObject *item);
 
-// Constructors
+// Temporal utility functions
+int is_leap_year(int year);
+long days_since_epoch(int year, int month, int day);
+int parse_iso_date(const char *str, Py_ssize_t len, int *year, int *month,
+                   int *day);
+int parse_iso_time(const char *str, Py_ssize_t len, int *hour, int *minute,
+                   int *second, int *microsecond);
+int parse_iso_timestamp(const char *str, Py_ssize_t len, int *year, int *month,
+                        int *day, int *hour, int *minute, int *second,
+                        int *microsecond, int *tz_offset_hours,
+                        int *tz_offset_minutes);
+
+PyObject *raypy_wrap_ray_object(obj_p ray_obj);
+
 PyObject *raypy_init_i16(PyObject *self, PyObject *args);
 PyObject *raypy_init_i32(PyObject *self, PyObject *args);
 PyObject *raypy_init_i64(PyObject *self, PyObject *args);
@@ -96,8 +115,6 @@ PyObject *raypy_init_list(PyObject *self, PyObject *args);
 PyObject *raypy_init_table(PyObject *self, PyObject *args);
 PyObject *raypy_init_dict(PyObject *self, PyObject *args);
 PyObject *raypy_init_vector(PyObject *self, PyObject *args);
-
-// Readers
 PyObject *raypy_read_i16(PyObject *self, PyObject *args);
 PyObject *raypy_read_i32(PyObject *self, PyObject *args);
 PyObject *raypy_read_i64(PyObject *self, PyObject *args);
@@ -111,52 +128,33 @@ PyObject *raypy_read_date(PyObject *self, PyObject *args);
 PyObject *raypy_read_time(PyObject *self, PyObject *args);
 PyObject *raypy_read_timestamp(PyObject *self, PyObject *args);
 PyObject *raypy_read_guid(PyObject *self, PyObject *args);
-
-// Type introspection
 PyObject *raypy_get_obj_type(PyObject *self, PyObject *args);
-
-// Table operations
 PyObject *raypy_table_keys(PyObject *self, PyObject *args);
 PyObject *raypy_table_values(PyObject *self, PyObject *args);
 PyObject *raypy_repr_table(PyObject *self, PyObject *args);
-
-// Dictionary operations
 PyObject *raypy_dict_keys(PyObject *self, PyObject *args);
 PyObject *raypy_dict_values(PyObject *self, PyObject *args);
 PyObject *raypy_dict_get(PyObject *self, PyObject *args);
-
-// Vector operations
 PyObject *raypy_at_idx(PyObject *self, PyObject *args);
 PyObject *raypy_insert_obj(PyObject *self, PyObject *args);
 PyObject *raypy_push_obj(PyObject *self, PyObject *args);
 PyObject *raypy_set_obj(PyObject *self, PyObject *args);
-PyObject *raypy_fill_vector(PyObject *self, PyObject *args);
-PyObject *raypy_fill_list(PyObject *self, PyObject *args);
-
-// Misc operations
 PyObject *raypy_get_obj_length(PyObject *self, PyObject *args);
 PyObject *raypy_eval_str(PyObject *self, PyObject *args);
 PyObject *raypy_get_error_obj(PyObject *self, PyObject *args);
 PyObject *raypy_binary_set(PyObject *self, PyObject *args);
-PyObject *raypy_env_get_internal_function_by_name(PyObject *self,
-                                                  PyObject *args);
-PyObject *raypy_env_get_internal_name_by_function(PyObject *self,
-                                                  PyObject *args);
+PyObject *raypy_env_get_internal_fn_by_name(PyObject *self, PyObject *args);
+PyObject *raypy_env_get_internal_name_by_fn(PyObject *self, PyObject *args);
 PyObject *raypy_eval_obj(PyObject *self, PyObject *args);
 PyObject *raypy_loadfn(PyObject *self, PyObject *args);
 PyObject *raypy_quote(PyObject *self, PyObject *args);
 PyObject *raypy_rc(PyObject *self, PyObject *args);
 PyObject *raypy_set_obj_attrs(PyObject *self, PyObject *args);
-
-// Database operations
-PyObject *raypy_select(PyObject *self, PyObject *args);
 PyObject *raypy_update(PyObject *self, PyObject *args);
 PyObject *raypy_insert(PyObject *self, PyObject *args);
 PyObject *raypy_upsert(PyObject *self, PyObject *args);
-
-// IO operations
 PyObject *raypy_hopen(PyObject *self, PyObject *args);
 PyObject *raypy_hclose(PyObject *self, PyObject *args);
 PyObject *raypy_write(PyObject *self, PyObject *args);
 
-#endif // RAYFORCE_C_H
+#endif