- 安装 JDK
- # tar zxvf jdk-8u281-linux-x64.tar.gz -C / opt /
- # vi / etc / profile
- #### zookeeper ####
export JAVA_HOME = / opt / jdk1.8.0_281
export CLASSPATH = $ JAVA_HOME / jre / lib: $ JAVA_HOME / lib
export PATH = $ JAVA_HOME / bin: $ PATH
- # source / etc / profile
- # java -version
version java "1.8.0_281"
Environnement d'exécution Java (TM) SE (build 1.8.0_281-b09)
VM serveur 64 bits Java HotSpot (TM) (build 25.281-b09, mode mixte)
- #
- Kafka
- # tar zxvf kafka_2.13-2.7.0.tgz -C / opt /
- # cd /opt/kafka_2.13-2.7.0/config
- # vi / etc / profile
- #### zookeeper ####
export JAVA_HOME = / opt / jdk1.8.0_281
export KAFKA_HOME = / opt / kafka_2.13-2.7.0
export CLASSPATH = $ JAVA_HOME / jre / lib: $ JAVA_HOME / lib: $ KAFKA_HOME / lib
export PATH = $ KAFKA_HOME / bin: $ JAVA_HOME / bin: $ PATH
- #### 启动 zookeeper #####
- # cd /opt/kafka_2.13-2.7.0/
- # nohup bin / zookeeper-server-start.sh config / zookeeper.properties &
- #### 启动 kafka #####
- # nohup bin / kafka-server-start.sh config / server.properties &
- ### 创建 topic ###
- # kafka-topics.sh --create --zookeeper localhost: 2181 --replication-factor 1 --partitions 1 --topic test Test du
sujet créé.
- #
- ### 显示 topic ###
- # kafka-topics.sh -list -zookeeper localhost: 2181
test
- #
Créez un producteur kafka:
- # kafka-console-producer.sh --broker-list localhost: 9092 --topic test
bonjour
test_msg
test_topic
Créez un consommateur kafka:
- # kafka-console-consumer.sh --bootstrap-server localhost: 9092 --topic test --from- starting
hello
test_msg
test_topic
Explication:
--broker-list: le port par défaut est 9092. Vous pouvez le modifier vous-même
- Préparation de la base de données
postgres = # créer l'utilisateur sync_user avec le mot de passe de réplication 'sync_user';
postgres = # modifier l'utilisateur sync_user avec le superutilisateur;
postgres = # créer la base de données tdb01;
CREATE DATABASE
postgres = # CREATE TABLE clients (
id SERIAL,
first_name VARCHAR (255) NOT NULL,
last_name VARCHAR (255) NOT NULL,
email VARCHAR (255) NOT NULL,
PRIMARY KEY (id)
);
pg_hba.conf
- # TYPE MÉTHODE D'ADRESSE UTILISATEUR BASE DE DONNÉES
- # "local" est pour les connexions de socket de domaine Unix uniquement
local all all trust
- # Connexions locales IPv4:
host all all 127.0.0.1/32 trust
host all all 0.0.0.0/0 md5
- # connexions locales IPv6:
host all all: : 1/128 trust
- # Autorise les connexions de réplication à partir de localhost, par un utilisateur avec le
privilège de réplication - #.
réplication locale tous confiance
réplication de l'hôte tous 127.0.0.1/32
réplication de l'hôte de confiance tous 192.168.1.0/24 md5
$
- ### Modification du fichier de configuration ####
$ vi postgresql.conf
wal_level = logical
#max_wal_senders = 10
#max_replication_slots = 10
- #### redémarrer la base de données ####
$ pg_ctl stop –D / pgdata
$ pg_ctl start –D / pgdata
- Configurer et installer Connect
- # tar zxvf debezium-connector-postgres-1.4.1.Final-plugin.tar.gz -C / opt /
- # cd / opt / debezium-connector-postgres /
- # cp * .jar /opt/kafka_2.13 -2.7.0 / libs /
- # cd /opt/kafka_2.13-2.7.0/config
- # vi kafka-postgres.properties
name = postgres-connector
connector.class = io.debezium.connector.postgresql.PostgresConnector
base de données. hostname = 192.168.1.141
database.port = 5432
database.user = sync_user
database.password = sync_user
database.dbname = tdb01
database.server.name = tdb01
plugin.name = pgoutput
table.whitelist =
errors.log.enable = true
errors. logs.include.messages = true
- #
- Test d'insertion de données de base de données
- #### 数据库 端 ####
postgres = # \ c tdb01
tdb01 = # insérer dans les valeurs des clients (prénom, nom, e-mail) ('he', 'mia', '[email protected]');
INSÉRER 0 1
tdb01 = #
- #### bord kafka #####
- # kafka-topics.sh -list -zookeeper localhost: 2181
tdb01.public.customers
- # kafka-console-consumer.sh --bootstrap-server localhost: 9092 --topic tdb01.public.customers --from-begin
{"schema": {"type": "struct", "fields": [{"type": "struct", "fields": [{"type": "int32", "optional": false, "field ":" id "}, {" type ":" string "," optional ": false," field ":" first_name "}, {" type ":" string "," optional ": false," field ": "last_name"}, {"type": "string", "optional": false, "field": "email"}], "optional": true, "name": "tdb01.public.customers.Value", "field": "before"}, {"type": "struct", "fields": [{"type": "int32", "optional": false, "field": "id"}, {"type ":"chaîne","optionnel": false, "field": "first_name"}, {"type": "string", "optional": false, "field": "last_name"}, {"type": "string", "optional ": false," field ":" email "}]," optional ": true," name ":" tdb01.public.customers.Value "," field ":" after "}, {" type ":" struct "," fields ": [{" type ":" string "," optional ": false," field ":" version "}, {" type ":" string "," optional ": false," field ": "connector"}, {"type": "string", "optional": false, "field": "name"}, {"type": "int64", "optional": false, "field": "ts_ms "}, {"type": "string", "optional": true, "name": "io.debezium.data.Enum", "version": 1, "parameters": {"allowed": "true, last , false "}," default ":" false "," field ":" snapshot "}, {" type ":" string "," optional ": false," field ":" db "}, {" type " : "string", "optional": false, "field": "schema"}, {"type": "string", "optional": false, "field": "table"}, {"type": " int64 "," optionnel ": true," field ":" txId "}, {" type ":" int64 "," optional ": true," field ":" lsn "}, {" type ":" int64 " , "facultatif": vrai, "field ":" xmin "}]," optional ": false," name ":" io.debezium.connector.postgresql.Source "," field ":" source "}, {" type ":" string "," optionnel ": false," field ":" op "}, {" type ":" int64 "," optional ": true," field ":" ts_ms "}, {" type ":" struct "," fields " : [{"type": "string", "optional": false, "field": "id"}, {"type": "int64", "optional": false, "field": "total_order"}, {"type": "int64", "optional": false, "field": "data_collection_order"}], "optional": true, "field": "transaction"}], "optional": false, "name" : "tdb01.public.customers.Envelope "}," payload ": {" before ": null," after ": {" id ": 1," first_name ":" he "," last_name ":" mia "," email ":" [email protected] "}," source ": {" version ":" 1.4.1.Final "," connector ":" postgresql "," name ":" tdb01 "," ts_ms ": 1612544437937, "snapshot": "false", "db": "tdb01", "schema": "public", "table": "clients", "txId": 493, "lsn": 23092184, "xmin": null} , "op": "c", "ts_ms": 1612515639176, "transaction": null}}first_name ":" he "," last_name ":" mia "," email ":" [email protected] "}," source ": {" version ":" 1.4.1.Final "," connector ":" postgresql "," name ":" tdb01 "," ts_ms ": 1612544437937," snapshot ":" false "," db ":" tdb01 "," schema ":" public "," table ":" clients "," txId ": 493," lsn ": 23092184," xmin ": null}," op ":" c "," ts_ms ": 1612515639176," transaction ": null}}first_name ":" he "," last_name ":" mia "," email ":" [email protected] "}," source ": {" version ":" 1.4.1.Final "," connector ":" postgresql "," name ":" tdb01 "," ts_ms ": 1612544437937," snapshot ":" false "," db ":" tdb01 "," schema ":" public "," table ":" customers "," txId ": 493," lsn ": 23092184," xmin ": null}," op ":" c "," ts_ms ": 1612515639176," transaction ": null}}ts_ms ": 1612544437937," snapshot ":" false "," db ":" tdb01 "," schema ":" public "," table ":" clients "," txId ": 493," lsn ": 23092184," xmin ": null}," op ":" c "," ts_ms ": 1612515639176," transaction ": null}}ts_ms ": 1612544437937," snapshot ":" false "," db ":" tdb01 "," schema ":" public "," table ":" clients "," txId ": 493," lsn ": 23092184," xmin ": null}," op ":" c "," ts_ms ": 1612515639176," transaction ": null}}