Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
164 commits
Select commit Hold shift + click to select a range
72cc4db
[UOD-941] Add compaction cli property (#1)
romain-gilles-ultra Sep 28, 2021
b3980c3
Bugfix/uod 941 fix ci docker build (#2)
romain-gilles-ultra Sep 29, 2021
b049503
fix docker build stage issue
romain-gilles-ultra Sep 29, 2021
a11deff
[UOD-941] Add kafka-message-max-bytes configuration option to support…
romain-gilles-ultra Oct 12, 2021
3b24ffd
[UOD-941] Add missing message max bytes option (#5)
romain-gilles-ultra Oct 14, 2021
582d30c
Fix gitlab run pipeline on tag
romain-gilles-ultra Oct 15, 2021
1f81777
Fix gitlab run pipeline on tag
romain-gilles-ultra Oct 15, 2021
9c3692c
rework mapping, add draft of correlation, add capture to prepare unit…
romain-gilles-ultra Dec 23, 2021
04ebabc
Add unit test on the mapping function
romain-gilles-ultra Dec 28, 2021
5e0be95
Add benchmark on mapping function
romain-gilles-ultra Jan 3, 2022
0efcadd
Rework transform method to make it more functional
romain-gilles-ultra Jan 3, 2022
44e749d
add make bench commands
romain-gilles-ultra Jan 3, 2022
18136cd
rename mapper to adapter
romain-gilles-ultra Jan 3, 2022
5021147
complite adapt method test
romain-gilles-ultra Jan 3, 2022
7869900
rework CEL activation and add db_ops
romain-gilles-ultra Jan 3, 2022
e98724d
updated the documentation with the new db_ops field
romain-gilles-ultra Jan 4, 2022
ec5400c
updated documentation
romain-gilles-ultra Jan 4, 2022
47c4d17
improve bench output
romain-gilles-ultra Jan 12, 2022
e2b9614
improve adapter performance
romain-gilles-ultra Jan 12, 2022
7f55b8d
remove unused feature: extensions
romain-gilles-ultra Jan 17, 2022
fcc3e9c
perpare operations dbops
romain-gilles-ultra Jan 19, 2022
b4c5513
integrate iso feature operations dbops
romain-gilles-ultra Jan 19, 2022
369d826
integrate actions based operations on dbops
romain-gilles-ultra Jan 19, 2022
d297618
add table expression on action operation
romain-gilles-ultra Jan 19, 2022
826c3df
update documentation and add source to the ce_id
romain-gilles-ultra Jan 20, 2022
07809cf
update documentation and add remove group function replace it with a …
romain-gilles-ultra Jan 21, 2022
657961d
add avro schema generation
romain-gilles-ultra Jan 27, 2022
766fb9c
add more avro schema generation: tables, actions
romain-gilles-ultra Jan 27, 2022
7c72edc
draft implementation on CDC
romain-gilles-ultra Feb 1, 2022
5a72466
fix abi struct inheritance
romain-gilles-ultra Feb 1, 2022
3367d81
add new benchmarks
romain-gilles-ultra Feb 1, 2022
cc47fbd
rework command structure
romain-gilles-ultra Feb 2, 2022
b86da69
update dependencies
romain-gilles-ultra Feb 2, 2022
558b570
improve cdc table message key
romain-gilles-ultra Feb 2, 2022
992d17e
add support for avro codec with schema registry
romain-gilles-ultra Mar 8, 2022
6f6fcc4
add support for avro codec with schema registry
romain-gilles-ultra Mar 8, 2022
f943d89
add support for avro codec with schema registry
romain-gilles-ultra Mar 8, 2022
9059bb6
add support for avro codec with schema registry
romain-gilles-ultra Mar 8, 2022
b0a2f1c
add support for avro codec with schema registry
romain-gilles-ultra Mar 8, 2022
7559d7f
add support for avro codec with schema registry
romain-gilles-ultra Mar 8, 2022
9ed40a1
add support for avro codec with schema registry
romain-gilles-ultra Mar 9, 2022
fa97e86
add support for avro codec with schema registry
romain-gilles-ultra Mar 11, 2022
a8bfa92
fix CI
romain-gilles-ultra Mar 11, 2022
f12821f
[UOD-105] add correlation id (#6)
romain-gilles-ultra Mar 16, 2022
ce72a92
exclude inject & fixaccount actions in cdc table mode
copoix-ultra Mar 16, 2022
dd72662
improve checkpoint management and context
romain-gilles-ultra Mar 16, 2022
d4709b0
improve save checkpoint cursor log
romain-gilles-ultra Mar 16, 2022
2db8b32
Rework the cursor management to avoid transaction and a second topic
romain-gilles-ultra Mar 25, 2022
031e4bd
handle ABI binary extensions
copoix-ultra Mar 26, 2022
c274301
fix namespace case (#8)
copoix-ultra Mar 26, 2022
a62e6b6
Fix codec cache
romain-gilles-ultra Mar 26, 2022
5cc7e05
Add key extractor/selector for cdc
romain-gilles-ultra Mar 28, 2022
585f522
reinclude inject action in cdc table mode
copoix-ultra Mar 28, 2022
f8283bd
Merge branch 'main' of github.com:ultraio/dkafka
copoix-ultra Mar 28, 2022
9833b3e
improve avro to support converters
romain-gilles-ultra Mar 29, 2022
6acf021
add avro converter
romain-gilles-ultra Mar 29, 2022
c01be41
remove wrong files
romain-gilles-ultra Mar 29, 2022
a975cdd
start to make asset type works in avro
romain-gilles-ultra Mar 29, 2022
83930a5
start to make asset type works in avro
romain-gilles-ultra Mar 29, 2022
8fe582c
start to make asset type works in avro
romain-gilles-ultra Mar 29, 2022
57b06d0
Add header test and update backlog
romain-gilles-ultra Mar 30, 2022
a79f46a
Rework command line options
romain-gilles-ultra Apr 5, 2022
9241b96
Rework command line options
romain-gilles-ultra Apr 5, 2022
235756c
Add missing content-type header to the DKafkaCheckPoint message
romain-gilles-ultra Apr 14, 2022
e493595
Fix load cursor position on empty topic
romain-gilles-ultra Apr 14, 2022
85c709f
Add legacy JSON format option for the asset and use it in when publis…
romain-gilles-ultra Apr 25, 2022
c092750
improve date format and ce_id computation
romain-gilles-ultra May 30, 2022
0db399d
Add the step name to the ce_id computation to distinguish New from Undo
romain-gilles-ultra Jun 3, 2022
bb03fb9
add basic table operations info in the cdc action message
romain-gilles-ultra Jun 17, 2022
cdce3f2
Update .gitlab-ci.yml file to find the docker host
flassagn Jun 21, 2022
0eb010a
added eosio.nft.ft-2.0 abi
copoix-ultra Jun 23, 2022
f8c9729
Fix schema generation by using the name instead of the type
romain-gilles-ultra Jun 27, 2022
40e63a6
remove omitempty on structs fields
copoix-ultra Jun 27, 2022
b6e7847
Add avro serialization for DKafkaCheckpoint message
romain-gilles-ultra Jul 5, 2022
b230efa
add block time to the cdc messages in the block context
romain-gilles-ultra Jul 5, 2022
fdf90d0
Change blockchain time type representation in avro to support timesta…
romain-gilles-ultra Jul 12, 2022
ee5b8e9
Add cursor string to the cdc context
romain-gilles-ultra Jul 12, 2022
1c15e56
Minor rename and config
romain-gilles-ultra Jul 29, 2022
2fdb993
[UOD-1169] add new mapping for db ops index (#9)
Aug 25, 2022
00ef722
remove fixaccount hack
copoix-ultra Aug 25, 2022
a1ac9a5
fix schema meta field bad name on compatibility
romain-gilles-ultra Aug 31, 2022
741b4e2
remove kafka message timestamp to use ingestion time instead of block…
romain-gilles-ultra Aug 31, 2022
067a3e2
remove kafka message timestamp to use ingestion time instead of block…
romain-gilles-ultra Aug 31, 2022
af945c4
[UOD-1171] Adding abi block number as part of schema versioning (#10)
Sep 1, 2022
15fe116
[UOD-1225] force subject to forward compatible
romain-gilles-ultra Sep 26, 2022
1595332
Update docker compose with redpanda
romain-gilles-ultra Sep 27, 2022
bd41961
[BLOCK-652] Check action trace receipt when setting global sequence (…
Duncan-Ultra Sep 30, 2022
d700203
[UOD-1196] imply reverse order on undo step (#12)
romain-gilles-ultra Nov 20, 2022
aa56099
[UOD-1290] Add support for dkafka legacy cursor topic migration (#13)
romain-gilles-ultra Nov 23, 2022
e009b6e
[UOD-1172] Add support of ABI auto refresh (#14)
romain-gilles-ultra Nov 24, 2022
4227e2a
[UOD-1224] support wildcard for dynamic cdc tables filtering (#15)
romain-gilles-ultra Nov 25, 2022
8e82aa7
[UOD-1291] improve dkafka resilience (#16)
romain-gilles-ultra Nov 28, 2022
1610f5e
[UOD-1292] add source and domain properties to schema's meta (#17)
romain-gilles-ultra Nov 28, 2022
4ec1ef6
[UOD-1296] remove cursor topic auto-create (#18)
romain-gilles-ultra Dec 2, 2022
8c90963
[UOD-1297] make sure kafka timestamp is the ingestion time and not th…
romain-gilles-ultra Dec 2, 2022
9a6fbf4
[UOD-1299] add wildcard support on cdc actions (#20)
romain-gilles-ultra Dec 14, 2022
1a7bee1
[UOD-1324] add abi read command as a debug feature (#21)
romain-gilles-ultra Dec 20, 2022
e1e3665
[UOD-1325] map binary extension type to optional type (#22)
romain-gilles-ultra Dec 20, 2022
972fb7e
[UOD-1290] add topic cursor migration to the publish command (#23)
romain-gilles-ultra Dec 21, 2022
007902c
[UOD-1329] improve dkafka version migration on cursors discovery (#24)
romain-gilles-ultra Dec 21, 2022
bdd20c5
[GP-17941] add logs to track processing perf
romain-gilles-ultra Feb 3, 2023
2bbef1f
[UOD-1382] properly handle the delivery message status (#25)
romain-gilles-ultra Feb 24, 2023
989218f
[UOD-1382] fix stuck process on error
romain-gilles-ultra Feb 28, 2023
dbce855
Merge pull request #26 from ultraio/bugfix/UOD-1382_fix_stuck_process…
anaferreira2 Feb 28, 2023
8796b13
[UOD-1406] fix abi auto-refresh (#27)
romain-gilles-ultra Mar 1, 2023
658e416
[INTERNAL] update docker compose images
romain-gilles-ultra May 10, 2023
b6115f5
[UDO-1228] update eos and dfuse-eos dependencies (#28)
romain-gilles-ultra Jul 24, 2023
28ae930
[UOD-1653] Providing support for numeric boolean
Aug 18, 2023
e69c6cc
Adding test
Aug 21, 2023
4b707e3
[UOD-1653] fixing test build errors
romain-gilles-ultra Aug 21, 2023
24fc458
[UOD-1653] fixing boolean use case
romain-gilles-ultra Aug 21, 2023
7bbf2d3
Merge pull request #29 from ultraio/bugfix/UOD-1653-dkafka_crashes_wi…
Aug 21, 2023
cf5e0eb
[UOD-1695] support more blockchain types (#30)
romain-gilles-ultra Sep 22, 2023
c1fa258
[UOD-1695] add variant support and uint128, int128 (#31)
romain-gilles-ultra Sep 26, 2023
eaa4a12
[UOD-1694] add transaction cdc
romain-gilles-ultra Sep 27, 2023
e59f8b2
[UOD-1694] add transaction cdc
romain-gilles-ultra Sep 27, 2023
f3821c8
[UOD-1695] handle setabi when tracking eosio smart contract
romain-gilles-ultra Sep 28, 2023
a3f486d
[UOD-1695] handle setabi when tracking eosio smart contract
romain-gilles-ultra Sep 28, 2023
e56ccf6
Merge pull request #32 from ultraio/bugfix/UOD-1695_handle_setabi_on_…
Sep 28, 2023
3cf6dcd
[UOD-1694] Adding transaction mappings
Sep 29, 2023
9cf176f
[UOD-1694] Adding test
Oct 3, 2023
7b88d20
[UOD-1694] Fixing block header
Oct 3, 2023
c492d0a
[UOD-1694] Fixing block header
Oct 3, 2023
d95f07d
[UOD-1694] Updating from main
Oct 3, 2023
009e250
[UOD-1694] remove redundant fields
romain-gilles-ultra Oct 3, 2023
3fd2659
Merge pull request #33 from ultraio/task/UOD-1694_transaction_notific…
Oct 3, 2023
91e50e6
fix transaction notification avro schema
romain-gilles-ultra Oct 3, 2023
cd99aed
[UOD-1727] fix symbol and uint64 to avro mapping
romain-gilles-ultra Oct 6, 2023
7d8996d
[UOD-1727] fix symbol and uint64 to avro mapping
romain-gilles-ultra Oct 6, 2023
a47ca31
Merge pull request #34 from ultraio/bugfix/UOD-1727_fix_symbol_uint64…
Oct 6, 2023
7ff4073
[UOD-1685] Adding major version to abi versioning
Oct 9, 2023
0373e06
[UOD-1685] Adding major version to abi versioning
Oct 9, 2023
eed4dd9
Merge pull request #35 from ultraio/bugfix/UOD-1685-create-view-missi…
Oct 9, 2023
4d44434
wip
Oct 10, 2023
5e8b16f
Merge pull request #36 from ultraio/bugfix/UOD-1685-create-view-missi…
Oct 10, 2023
c97d3eb
[INTERNAL] register transaction schema only on transaction command (#37)
romain-gilles-ultra Oct 19, 2023
541d94a
[INTERNAL] apply feedback on transaction schema (#38)
romain-gilles-ultra Oct 19, 2023
515007d
[UOD-1807] hotfix of uint64 serialization when greater than int64 (#39)
romain-gilles-ultra Nov 17, 2023
b7b9ec5
[UOD-1728] fix symbol converter (#40)
romain-gilles-ultra Dec 7, 2023
c46cfd9
[UOD-1854] better support eos boolean
romain-gilles-ultra Dec 14, 2023
da951dd
Merge pull request #41 from ultraio/bugfix/UOD-1854_better_support_eo…
Dec 14, 2023
2e8d6e2
[UOD-1887] add eos.type property to primitive avro types (#42)
romain-gilles-ultra Jan 10, 2024
58bc477
[UOD-1887] add logical type for unit64 as some lib does not support c…
romain-gilles-ultra Jan 11, 2024
e453ed3
[UOD-1896] update eos-go version (#44)
romain-gilles-ultra Jan 17, 2024
1b4587d
[UOD-1935] use 13 dotted value on empty name for message keys (#45)
romain-gilles-ultra Jan 22, 2024
6945c3b
Introduce nix settings to setup go environement (#47)
olivier-norture-ultra Oct 23, 2024
b56c39d
Introduce TypedSchema (#48)
olivier-norture-ultra Oct 23, 2024
311c22d
Add automatic avro to pojo generation (#49)
olivier-norture-ultra Oct 23, 2024
798a343
Fix variant usages in union (#46)
olivier-norture-ultra Oct 23, 2024
1ffa567
Add uber variant type (#50)
olivier-norture-ultra Oct 23, 2024
8861332
Update abi to remove bool type in uber variant (#51)
olivier-norture-ultra Oct 25, 2024
bb479ec
Make eos.type optional in TypeSchema (#52)
olivier-norture-ultra Oct 25, 2024
968e578
[UOD-2194] uber variant serialization error (#53)
romain-gilles-ultra Oct 30, 2024
6179b0b
[UOD-2196] Handle string values for Union (#54)
olivier-norture-ultra Oct 31, 2024
79442cd
[UOD-2200] Add action tree information (#55)
olivier-norture-ultra Nov 5, 2024
e818d5f
[INTERNAL] Estimate ultra rgrab message size (#56)
olivier-norture-ultra Dec 3, 2024
6bf9c4e
[UOD-2255] fix issue on CEL first_auth_actor when action has no auth …
romain-gilles-ultra Jan 17, 2025
ca1fb73
[UOD-2318] Add support for notifications on cdc tables (#58)
romain-gilles-ultra Mar 3, 2025
3ff7e10
[UOD-2349] fix docker image build (#59)
romain-gilles-ultra Apr 2, 2025
073e868
[UOD-2375] fix inline events namespace generation and add support for…
romain-gilles-ultra Jun 2, 2025
b38596b
[UOD-2377] add skip-dbops option on cdc action (#61)
romain-gilles-ultra Jun 2, 2025
cc181b8
[UOD-2378] fix duplicate declaration of Asset when used with Extended…
romain-gilles-ultra Jun 3, 2025
8c721a2
[UOD-2381] add subject compatibility option management defaulted to F…
romain-gilles-ultra Jun 11, 2025
e5f9289
[UOD-2383] remove source from schema meta
romain-gilles-ultra Jun 24, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
build
coverage
coverage.cov
coverage.html
.idea
new.txt
.vscode
*.pem
avro-tools.jar
.aider*
.env
112 changes: 112 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# see https://docs.gitlab.com/ee/ci/caching/index.html#cache-go-dependencies

cache:
paths:
- /apt-cache
- /go/src/github.com
- /go/src/golang.org
- /go/src/google.golang.org
- /go/src/gopkg.in

stages:
- test
- build
- image

.golang_based:
tags:
- docker
image: golang:1.24-bookworm

unit-tests:
extends:
- .golang_based
stage: test
script:
- make test

# race-detector:
# extends:
# - .golang_based
# stage: test
# script:
# - make race

# memory-sanitizer:
# extends:
# - .golang_based
# stage: test
# script:
# - make msan

code-coverage:
extends:
- .golang_based
stage: test
script:
- make cov

code-coverage-report:
extends:
- .golang_based
stage: test
script:
- make covhtml
rules:
- if: '$CI_COMMIT_BRANCH == "main"'

# lint_code:
# extends:
# - .golang_based
# stage: test
# script:
# - make lint

build:
extends:
- .golang_based
stage: build
script:
- make
artifacts:
paths:
- dkafka

build-image:
tags:
- docker
stage: image
variables:
DOCKER_HOST: "tcp://localhost:2375"
image: docker:git
services:
- docker:19.03.12-dind
script:
- docker build --no-cache --tag "quay.io/ultraio/dkafka:latest" .
cache:
paths: []

build-publish-docker:
tags:
- docker
stage: image
variables:
VERSION: "${CI_COMMIT_SHORT_SHA}"
DOCKER_HOST: "tcp://localhost:2375"
image: docker:git
services:
- docker:19.03.12-dind
rules:
- if: $CI_COMMIT_TAG
variables:
VERSION: "${CI_COMMIT_TAG}"
- if: '$CI_COMMIT_BRANCH == "main"'
- if: '$CI_COMMIT_BRANCH == "develop"'
script:
- docker login "quay.io" --username "${QUAY_USERNAME}" --password "${QUAY_PASSWORD}"
- docker build --no-cache --tag "quay.io/ultraio/dkafka:${VERSION}" .
- docker push "quay.io/ultraio/dkafka:${VERSION}"
- docker tag "quay.io/ultraio/dkafka:${VERSION}" "quay.io/ultraio/dkafka:latest"
- docker push "quay.io/ultraio/dkafka:latest"
cache:
paths: []
15 changes: 15 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Attach to Process",
"type": "go",
"request": "attach",
"mode": "local",
"processId": 0
}
]
}
59 changes: 59 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{
"cSpell.words": [
"abicodec",
"avsc",
"bstream",
"cloudevent",
"covhtml",
"DBOP",
"dbops",
"dedupe",
"derr",
"dfuse",
"dgrpc",
"dkafka",
"eewew",
"eoscanada",
"eosio",
"goavro",
"gopkg",
"gotest",
"GRPC",
"jsonpb",
"memoizable",
"mkdir",
"msan",
"pantomath",
"pbabicodec",
"pbbstream",
"pbcodec",
"pbgo",
"pbhealth",
"protobuf",
"ptypes",
"ricardian",
"riferrei",
"setabi",
"specversion",
"srclient",
"SSLCA",
"Structs",
"trxs",
"Undecodable",
"varuint",
"zlog",
"zstd"
],
"nixEnvSelector.nixFile": "${workspaceFolder}/shell.nix",
"terminal.integrated.profiles.linux": {
"BashWithStartup": {
"path": "bash",
"args": [
"-c",
"nix-shell"
]
}
},
"terminal.integrated.defaultProfile.linux": "BashWithStartup",
"makefile.configureOnOpen": false
}
25 changes: 25 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
FROM golang:1.24-bookworm AS build

WORKDIR /app

COPY go.mod ./
COPY go.sum ./
COPY fork ./fork
COPY action ./action
COPY table ./table
COPY *.go ./
COPY cmd ./cmd

RUN go mod download

RUN go build -o /dkafka -v ./cmd/dkafka

FROM gcr.io/distroless/base-debian12

WORKDIR /

COPY --from=build /dkafka /dkafka

USER nonroot:nonroot

ENTRYPOINT ["/dkafka"]
Loading