Skip to content

Commit 03954b6

Browse files
committed
WIP LMDB implementation
1 parent 1fcbbfb commit 03954b6

File tree

6 files changed

+264
-0
lines changed

6 files changed

+264
-0
lines changed

.github/workflows/build-linux.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,14 @@ jobs:
2525
restore-keys: |
2626
${{ runner.os }}-konan-
2727
28+
- name: glibc install
29+
run: |
30+
sudo apt install glibc-source -y
31+
32+
- name: LMDB install
33+
run: |
34+
sudo apt-get install liblmdb-dev
35+
2836
- name: Linux build
2937
run: |
3038
./gradlew build publishToMavenLocal --no-daemon --stacktrace

multiplatform-settings/api/multiplatform-settings.klib.api

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,35 @@ final class com.russhwolf.settings/StorageSettings : com.russhwolf.settings/Sett
202202
final val size // com.russhwolf.settings/StorageSettings.size|{}size[0]
203203
final fun <get-size>(): kotlin/Int // com.russhwolf.settings/StorageSettings.size.<get-size>|<get-size>(){}[0]
204204
}
205+
// Targets: [linuxX64]
206+
final class com.russhwolf.settings/LmdbSettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/LmdbSettings|null[0]
207+
constructor <init>(kotlin/String) // com.russhwolf.settings/LmdbSettings.<init>|<init>(kotlin.String){}[0]
208+
final fun clear() // com.russhwolf.settings/LmdbSettings.clear|clear(){}[0]
209+
final fun getBoolean(kotlin/String, kotlin/Boolean): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.getBoolean|getBoolean(kotlin.String;kotlin.Boolean){}[0]
210+
final fun getBooleanOrNull(kotlin/String): kotlin/Boolean? // com.russhwolf.settings/LmdbSettings.getBooleanOrNull|getBooleanOrNull(kotlin.String){}[0]
211+
final fun getDouble(kotlin/String, kotlin/Double): kotlin/Double // com.russhwolf.settings/LmdbSettings.getDouble|getDouble(kotlin.String;kotlin.Double){}[0]
212+
final fun getDoubleOrNull(kotlin/String): kotlin/Double? // com.russhwolf.settings/LmdbSettings.getDoubleOrNull|getDoubleOrNull(kotlin.String){}[0]
213+
final fun getFloat(kotlin/String, kotlin/Float): kotlin/Float // com.russhwolf.settings/LmdbSettings.getFloat|getFloat(kotlin.String;kotlin.Float){}[0]
214+
final fun getFloatOrNull(kotlin/String): kotlin/Float? // com.russhwolf.settings/LmdbSettings.getFloatOrNull|getFloatOrNull(kotlin.String){}[0]
215+
final fun getInt(kotlin/String, kotlin/Int): kotlin/Int // com.russhwolf.settings/LmdbSettings.getInt|getInt(kotlin.String;kotlin.Int){}[0]
216+
final fun getIntOrNull(kotlin/String): kotlin/Int? // com.russhwolf.settings/LmdbSettings.getIntOrNull|getIntOrNull(kotlin.String){}[0]
217+
final fun getLong(kotlin/String, kotlin/Long): kotlin/Long // com.russhwolf.settings/LmdbSettings.getLong|getLong(kotlin.String;kotlin.Long){}[0]
218+
final fun getLongOrNull(kotlin/String): kotlin/Long? // com.russhwolf.settings/LmdbSettings.getLongOrNull|getLongOrNull(kotlin.String){}[0]
219+
final fun getString(kotlin/String, kotlin/String): kotlin/String // com.russhwolf.settings/LmdbSettings.getString|getString(kotlin.String;kotlin.String){}[0]
220+
final fun getStringOrNull(kotlin/String): kotlin/String? // com.russhwolf.settings/LmdbSettings.getStringOrNull|getStringOrNull(kotlin.String){}[0]
221+
final fun hasKey(kotlin/String): kotlin/Boolean // com.russhwolf.settings/LmdbSettings.hasKey|hasKey(kotlin.String){}[0]
222+
final fun putBoolean(kotlin/String, kotlin/Boolean) // com.russhwolf.settings/LmdbSettings.putBoolean|putBoolean(kotlin.String;kotlin.Boolean){}[0]
223+
final fun putDouble(kotlin/String, kotlin/Double) // com.russhwolf.settings/LmdbSettings.putDouble|putDouble(kotlin.String;kotlin.Double){}[0]
224+
final fun putFloat(kotlin/String, kotlin/Float) // com.russhwolf.settings/LmdbSettings.putFloat|putFloat(kotlin.String;kotlin.Float){}[0]
225+
final fun putInt(kotlin/String, kotlin/Int) // com.russhwolf.settings/LmdbSettings.putInt|putInt(kotlin.String;kotlin.Int){}[0]
226+
final fun putLong(kotlin/String, kotlin/Long) // com.russhwolf.settings/LmdbSettings.putLong|putLong(kotlin.String;kotlin.Long){}[0]
227+
final fun putString(kotlin/String, kotlin/String) // com.russhwolf.settings/LmdbSettings.putString|putString(kotlin.String;kotlin.String){}[0]
228+
final fun remove(kotlin/String) // com.russhwolf.settings/LmdbSettings.remove|remove(kotlin.String){}[0]
229+
final val keys // com.russhwolf.settings/LmdbSettings.keys|{}keys[0]
230+
final fun <get-keys>(): kotlin.collections/Set<kotlin/String> // com.russhwolf.settings/LmdbSettings.keys.<get-keys>|<get-keys>(){}[0]
231+
final val size // com.russhwolf.settings/LmdbSettings.size|{}size[0]
232+
final fun <get-size>(): kotlin/Int // com.russhwolf.settings/LmdbSettings.size.<get-size>|<get-size>(){}[0]
233+
}
205234
// Targets: [mingwX64]
206235
final class com.russhwolf.settings/RegistrySettings : com.russhwolf.settings/Settings { // com.russhwolf.settings/RegistrySettings|null[0]
207236
constructor <init>(kotlin/String) // com.russhwolf.settings/RegistrySettings.<init>|<init>(kotlin.String){}[0]

multiplatform-settings/build.gradle.kts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl
1717
* limitations under the License.
1818
*/
1919

20+
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
21+
2022
plugins {
2123
id("standard-configuration")
2224
id("module-publication")
@@ -27,6 +29,13 @@ standardConfig {
2729
}
2830

2931
kotlin {
32+
targets.getByName<KotlinNativeTarget>("linuxX64") {
33+
compilations["main"].cinterops.create("lmdb")
34+
binaries.configureEach {
35+
// lmdb appears to be using a newer gcc than Kotlin. This lets us still work as long as host has newer gcc too
36+
linkerOpts += "--allow-shlib-undefined"
37+
}
38+
}
3039
@OptIn(ExperimentalKotlinGradlePluginApi::class)
3140
compilerOptions {
3241
freeCompilerArgs.add("-Xexpect-actual-classes")
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
* Copyright 2022 Russell Wolf
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.russhwolf.settings
18+
19+
import cnames.structs.MDB_cursor
20+
import cnames.structs.MDB_env
21+
import cnames.structs.MDB_txn
22+
import com.russhwolf.settings.cinterop.lmdb.MDB_CREATE
23+
import com.russhwolf.settings.cinterop.lmdb.MDB_NOTFOUND
24+
import com.russhwolf.settings.cinterop.lmdb.MDB_SUCCESS
25+
import com.russhwolf.settings.cinterop.lmdb.MDB_cursor_op
26+
import com.russhwolf.settings.cinterop.lmdb.MDB_dbi
27+
import com.russhwolf.settings.cinterop.lmdb.MDB_dbiVar
28+
import com.russhwolf.settings.cinterop.lmdb.MDB_stat
29+
import com.russhwolf.settings.cinterop.lmdb.MDB_val
30+
import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_close
31+
import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_del
32+
import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_get
33+
import com.russhwolf.settings.cinterop.lmdb.mdb_cursor_open
34+
import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_close
35+
import com.russhwolf.settings.cinterop.lmdb.mdb_dbi_open
36+
import com.russhwolf.settings.cinterop.lmdb.mdb_del
37+
import com.russhwolf.settings.cinterop.lmdb.mdb_env_close
38+
import com.russhwolf.settings.cinterop.lmdb.mdb_env_create
39+
import com.russhwolf.settings.cinterop.lmdb.mdb_env_open
40+
import com.russhwolf.settings.cinterop.lmdb.mdb_get
41+
import com.russhwolf.settings.cinterop.lmdb.mdb_put
42+
import com.russhwolf.settings.cinterop.lmdb.mdb_stat
43+
import com.russhwolf.settings.cinterop.lmdb.mdb_strerror
44+
import com.russhwolf.settings.cinterop.lmdb.mdb_txn_begin
45+
import com.russhwolf.settings.cinterop.lmdb.mdb_txn_commit
46+
import kotlinx.cinterop.ByteVar
47+
import kotlinx.cinterop.CPointer
48+
import kotlinx.cinterop.ExperimentalForeignApi
49+
import kotlinx.cinterop.MemScope
50+
import kotlinx.cinterop.alloc
51+
import kotlinx.cinterop.allocPointerTo
52+
import kotlinx.cinterop.cstr
53+
import kotlinx.cinterop.memScoped
54+
import kotlinx.cinterop.ptr
55+
import kotlinx.cinterop.reinterpret
56+
import kotlinx.cinterop.toKString
57+
import kotlinx.cinterop.value
58+
import platform.posix.S_IRWXG
59+
import platform.posix.S_IRWXO
60+
import platform.posix.S_IRWXU
61+
import platform.posix.mkdir
62+
import kotlin.experimental.ExperimentalNativeApi
63+
64+
@OptIn(ExperimentalForeignApi::class)
65+
@ExperimentalSettingsImplementation
66+
public class LmdbSettings(private val path: String) : Settings {
67+
override val keys: Set<String>
68+
get() = buildSet {
69+
lmdbCursorTransaction { _, key ->
70+
add(key.mv_data!!.reinterpret<ByteVar>().toKString())
71+
}
72+
}
73+
74+
override val size: Int
75+
get() = lmdbTransaction { txn, dbi ->
76+
val stat = alloc<MDB_stat>()
77+
mdb_stat(txn, dbi, stat.ptr)
78+
stat.ms_entries.toInt()
79+
}
80+
81+
public override fun clear(): Unit = lmdbCursorTransaction { cursor, _ ->
82+
mdb_cursor_del(cursor, 0.toUInt())
83+
}
84+
85+
public override fun remove(key: String): Unit =
86+
lmdbTransaction { txn, dbi -> mdb_del(txn, dbi, createMdbVal(key).ptr, null).checkError(MDB_NOTFOUND) }
87+
88+
public override fun hasKey(key: String): Boolean = loadString(key) != null
89+
90+
public override fun putInt(key: String, value: Int): Unit = saveString(key, value.toString())
91+
public override fun getInt(key: String, defaultValue: Int): Int = getIntOrNull(key) ?: defaultValue
92+
public override fun getIntOrNull(key: String): Int? = loadString(key)?.toInt()
93+
94+
public override fun putLong(key: String, value: Long): Unit = saveString(key, value.toString())
95+
public override fun getLong(key: String, defaultValue: Long): Long = getLongOrNull(key) ?: defaultValue
96+
public override fun getLongOrNull(key: String): Long? = loadString(key)?.toLong()
97+
98+
public override fun putString(key: String, value: String): Unit = saveString(key, value)
99+
public override fun getString(key: String, defaultValue: String): String = getStringOrNull(key) ?: defaultValue
100+
public override fun getStringOrNull(key: String): String? = loadString(key)
101+
102+
public override fun putFloat(key: String, value: Float): Unit = saveString(key, value.toString())
103+
public override fun getFloat(key: String, defaultValue: Float): Float = getFloatOrNull(key) ?: defaultValue
104+
public override fun getFloatOrNull(key: String): Float? = loadString(key)?.toFloat()
105+
106+
public override fun putDouble(key: String, value: Double): Unit = saveString(key, value.toString())
107+
public override fun getDouble(key: String, defaultValue: Double): Double = getDoubleOrNull(key) ?: defaultValue
108+
public override fun getDoubleOrNull(key: String): Double? = loadString(key)?.toDouble()
109+
110+
public override fun putBoolean(key: String, value: Boolean): Unit = saveString(key, value.toString())
111+
public override fun getBoolean(key: String, defaultValue: Boolean): Boolean = getBooleanOrNull(key) ?: defaultValue
112+
public override fun getBooleanOrNull(key: String): Boolean? = loadString(key)?.toBoolean()
113+
114+
private fun saveString(key: String, value: String): Unit = lmdbTransaction { txn, dbi ->
115+
mdb_put(txn, dbi, createMdbVal(key).ptr, createMdbVal(value).ptr, 0.toUInt()).checkError()
116+
}
117+
118+
private fun loadString(key: String): String? = lmdbTransaction { txn, dbi ->
119+
val value = alloc<MDB_val>()
120+
mdb_get(txn, dbi, createMdbVal(key).ptr, value.ptr).checkError(MDB_NOTFOUND)
121+
value.mv_data?.reinterpret<ByteVar>()?.toKString()
122+
}
123+
124+
private fun lmdbCursorTransaction(action: MemScope.(cursor: CPointer<MDB_cursor>?, key: MDB_val) -> Unit) =
125+
lmdbTransaction { txn, dbi ->
126+
val cursor = allocPointerTo<MDB_cursor>()
127+
mdb_cursor_open(txn, dbi, cursor.ptr).checkError()
128+
val mdbKey = alloc<MDB_val>()
129+
val mdbValue = alloc<MDB_val>()
130+
mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_FIRST).checkError(
131+
MDB_NOTFOUND
132+
)
133+
while (true) {
134+
if (mdbKey.mv_data == null) {
135+
break
136+
} else {
137+
action(cursor.value, mdbKey)
138+
}
139+
mdbKey.mv_data = null
140+
mdbKey.mv_size = 0u
141+
mdb_cursor_get(cursor.value, mdbKey.ptr, mdbValue.ptr, MDB_cursor_op.MDB_NEXT).checkError(
142+
MDB_NOTFOUND
143+
)
144+
}
145+
mdb_cursor_close(cursor.value)
146+
147+
}
148+
149+
private fun MemScope.createMdbVal(string: String): MDB_val {
150+
val mdbVal = alloc<MDB_val>()
151+
val cstr = string.cstr
152+
mdbVal.mv_data = cstr.ptr
153+
mdbVal.mv_size = cstr.size.toULong()
154+
return mdbVal
155+
}
156+
157+
private inline fun <T> lmdbTransaction(action: MemScope.(txn: CPointer<MDB_txn>?, dbi: MDB_dbi) -> T): T =
158+
memScoped {
159+
val env = allocPointerTo<MDB_env>()
160+
mdb_env_create(env.ptr).checkError()
161+
val mode = (S_IRWXU or S_IRWXG or S_IRWXO).toUInt()
162+
mkdir(path, mode)
163+
mdb_env_open(env.value, path, 0.toUInt(), mode).checkError()
164+
165+
val txn = allocPointerTo<MDB_txn>()
166+
mdb_txn_begin(env.value, null, 0.toUInt(), txn.ptr).checkError()
167+
168+
val dbi = alloc<MDB_dbiVar>()
169+
mdb_dbi_open(txn.value, null, MDB_CREATE.toUInt(), dbi.ptr).checkError()
170+
171+
val out = action(txn.value, dbi.value)
172+
173+
mdb_dbi_close(env.value, dbi.value)
174+
mdb_txn_commit(txn.value).checkError()
175+
mdb_env_close(env.value)
176+
177+
out
178+
}
179+
180+
private fun Int.checkError(vararg expectedErrors: Int) {
181+
@OptIn(ExperimentalNativeApi::class)
182+
assert(this == MDB_SUCCESS || this in expectedErrors) { "Error: ${mdb_strerror(this)?.toKString()}" }
183+
}
184+
}
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Copyright 2022 Russell Wolf
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.russhwolf.settings
18+
19+
@OptIn(ExperimentalSettingsImplementation::class)
20+
public class LmdbSettingsTest
21+
: BaseSettingsTest(
22+
platformFactory = object : Settings.Factory {
23+
override fun create(name: String?): Settings = LmdbSettings(name ?: "lmdbTest")
24+
},
25+
hasListeners = false
26+
) {
27+
// TODO add test cases to verify that we write to the files we think we do
28+
29+
// TODO add cleanup methods so we don't leave test DBs lying around
30+
}
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
headers = lmdb.h
2+
package = com.russhwolf.settings.cinterop.lmdb
3+
compilerOpts = -I/usr/include -I/usr/include/x86_64-linux-gnu/
4+
linkerOpts = -llmdb -L/usr/lib -L/usr/lib/x86_64-linux-gnu/

0 commit comments

Comments
 (0)