diff --git a/docs/apidoc/KafkaAdminClient.rst b/docs/apidoc/KafkaAdminClient.rst index 09227e45b..78b2430e3 100644 --- a/docs/apidoc/KafkaAdminClient.rst +++ b/docs/apidoc/KafkaAdminClient.rst @@ -1,3 +1,90 @@ +KafkaAdminClient +================ + .. autoclass:: kafka.KafkaAdminClient :members: - :inherited-members: + +:class:`~kafka.KafkaAdminClient` is composed of functional mixins. The +sections below group its methods by area. + + +Topics +------ + +.. automethod:: kafka.KafkaAdminClient.list_topics +.. automethod:: kafka.KafkaAdminClient.describe_topics +.. automethod:: kafka.KafkaAdminClient.create_topics +.. automethod:: kafka.KafkaAdminClient.wait_for_topics +.. automethod:: kafka.KafkaAdminClient.delete_topics + + +Partitions +---------- + +.. automethod:: kafka.KafkaAdminClient.create_partitions +.. automethod:: kafka.KafkaAdminClient.delete_records +.. automethod:: kafka.KafkaAdminClient.elect_leaders +.. automethod:: kafka.KafkaAdminClient.alter_partition_reassignments +.. automethod:: kafka.KafkaAdminClient.list_partition_reassignments +.. automethod:: kafka.KafkaAdminClient.describe_topic_partitions +.. automethod:: kafka.KafkaAdminClient.list_partition_offsets + + +Configs +------- + +.. automethod:: kafka.KafkaAdminClient.describe_configs +.. automethod:: kafka.KafkaAdminClient.list_config_resources +.. automethod:: kafka.KafkaAdminClient.alter_configs +.. automethod:: kafka.KafkaAdminClient.reset_configs + + +Consumer Groups +--------------- + +.. automethod:: kafka.KafkaAdminClient.list_groups +.. automethod:: kafka.KafkaAdminClient.describe_groups +.. automethod:: kafka.KafkaAdminClient.list_group_offsets +.. automethod:: kafka.KafkaAdminClient.alter_group_offsets +.. automethod:: kafka.KafkaAdminClient.reset_group_offsets +.. automethod:: kafka.KafkaAdminClient.delete_group_offsets +.. automethod:: kafka.KafkaAdminClient.delete_groups +.. automethod:: kafka.KafkaAdminClient.remove_group_members + + +ACLs +---- + +.. automethod:: kafka.KafkaAdminClient.describe_acls +.. automethod:: kafka.KafkaAdminClient.create_acls +.. automethod:: kafka.KafkaAdminClient.delete_acls + + +Cluster +------- + +.. automethod:: kafka.KafkaAdminClient.describe_cluster +.. automethod:: kafka.KafkaAdminClient.describe_log_dirs +.. automethod:: kafka.KafkaAdminClient.alter_replica_log_dirs +.. automethod:: kafka.KafkaAdminClient.describe_metadata_quorum +.. automethod:: kafka.KafkaAdminClient.get_broker_version_data +.. automethod:: kafka.KafkaAdminClient.api_versions +.. automethod:: kafka.KafkaAdminClient.describe_features +.. automethod:: kafka.KafkaAdminClient.update_features + + +Transactions +------------ + +.. automethod:: kafka.KafkaAdminClient.list_transactions +.. automethod:: kafka.KafkaAdminClient.describe_transactions +.. automethod:: kafka.KafkaAdminClient.describe_producers +.. automethod:: kafka.KafkaAdminClient.abort_transaction +.. automethod:: kafka.KafkaAdminClient.find_hanging_transactions + + +Users +----- + +.. automethod:: kafka.KafkaAdminClient.alter_user_scram_credentials +.. automethod:: kafka.KafkaAdminClient.describe_user_scram_credentials diff --git a/docs/apidoc/KafkaConnection.rst b/docs/apidoc/KafkaConnection.rst deleted file mode 100644 index 6f5d3cf21..000000000 --- a/docs/apidoc/KafkaConnection.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. autoclass:: kafka.net.connection.KafkaConnection - :members: diff --git a/docs/apidoc/KafkaConnectionManager.rst b/docs/apidoc/KafkaConnectionManager.rst deleted file mode 100644 index 25943facd..000000000 --- a/docs/apidoc/KafkaConnectionManager.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. autoclass:: kafka.net.manager.KafkaConnectionManager - :members: diff --git a/docs/apidoc/ClusterMetadata.rst b/docs/apidoc/misc/ClusterMetadata.rst similarity index 52% rename from docs/apidoc/ClusterMetadata.rst rename to docs/apidoc/misc/ClusterMetadata.rst index a55940329..0c855fe98 100644 --- a/docs/apidoc/ClusterMetadata.rst +++ b/docs/apidoc/misc/ClusterMetadata.rst @@ -1,2 +1,6 @@ +ClusterMetadata +=============== + .. autoclass:: kafka.cluster.ClusterMetadata :members: + :undoc-members: diff --git a/docs/apidoc/misc/ConsumerRebalanceListener.rst b/docs/apidoc/misc/ConsumerRebalanceListener.rst new file mode 100644 index 000000000..9bb9a1c52 --- /dev/null +++ b/docs/apidoc/misc/ConsumerRebalanceListener.rst @@ -0,0 +1,10 @@ +ConsumerRebalanceListener +========================= + +.. autoclass:: kafka.ConsumerRebalanceListener + :members: + :undoc-members: + +.. autoclass:: kafka.AsyncConsumerRebalanceListener + :members: + :undoc-members: diff --git a/docs/apidoc/misc/IsolationLevel.rst b/docs/apidoc/misc/IsolationLevel.rst new file mode 100644 index 000000000..4afc21715 --- /dev/null +++ b/docs/apidoc/misc/IsolationLevel.rst @@ -0,0 +1,6 @@ +IsolationLevel +============== + +.. autoclass:: kafka.IsolationLevel + :members: + :undoc-members: diff --git a/docs/apidoc/OffsetAndMetadata.rst b/docs/apidoc/misc/OffsetAndMetadata.rst similarity index 58% rename from docs/apidoc/OffsetAndMetadata.rst rename to docs/apidoc/misc/OffsetAndMetadata.rst index 88a1eb936..86ea4440d 100644 --- a/docs/apidoc/OffsetAndMetadata.rst +++ b/docs/apidoc/misc/OffsetAndMetadata.rst @@ -1,2 +1,5 @@ +OffsetAndMetadata +================= + .. autoclass:: kafka.OffsetAndMetadata :members: diff --git a/docs/apidoc/misc/OffsetSpec.rst b/docs/apidoc/misc/OffsetSpec.rst new file mode 100644 index 000000000..42a48ac38 --- /dev/null +++ b/docs/apidoc/misc/OffsetSpec.rst @@ -0,0 +1,6 @@ +OffsetSpec +========== + +.. autoclass:: kafka.OffsetSpec + :members: + :undoc-members: diff --git a/docs/apidoc/misc/Partitioner.rst b/docs/apidoc/misc/Partitioner.rst new file mode 100644 index 000000000..1dfc77ac4 --- /dev/null +++ b/docs/apidoc/misc/Partitioner.rst @@ -0,0 +1,6 @@ +Partitioner +=========== + +.. autoclass:: kafka.partitioner.Partitioner + :members: + :undoc-members: diff --git a/docs/apidoc/misc/Serializer.rst b/docs/apidoc/misc/Serializer.rst new file mode 100644 index 000000000..393094b62 --- /dev/null +++ b/docs/apidoc/misc/Serializer.rst @@ -0,0 +1,18 @@ +Serializer +========== + +.. autoclass:: kafka.Serializer + :members: + :undoc-members: + +.. autoclass:: kafka.Deserializer + :members: + :undoc-members: + +.. autoclass:: kafka.JsonSerializer + :members: + :undoc-members: + +.. autoclass:: kafka.DefaultSerializer + :members: + :undoc-members: diff --git a/docs/apidoc/TopicPartition.rst b/docs/apidoc/misc/TopicPartition.rst similarity index 61% rename from docs/apidoc/TopicPartition.rst rename to docs/apidoc/misc/TopicPartition.rst index bbb5d8456..c82c4f5a1 100644 --- a/docs/apidoc/TopicPartition.rst +++ b/docs/apidoc/misc/TopicPartition.rst @@ -1,2 +1,5 @@ +TopicPartition +============== + .. autoclass:: kafka.TopicPartition :members: diff --git a/docs/apidoc/modules.rst b/docs/apidoc/modules.rst index 1a7f88a88..38e65ea4b 100644 --- a/docs/apidoc/modules.rst +++ b/docs/apidoc/modules.rst @@ -1,14 +1,101 @@ kafka-python API **************** +The :mod:`kafka` package exposes a small set of public classes for working +with Apache Kafka. The following sections group them by role; each class +links to its own API reference page. + + +kafka +===== + +Application code typically interacts with one of three top-level clients. +Each owns a background IO thread and a shared async networking layer. + +- :class:`~kafka.KafkaConsumer` - high-level, group-aware message consumer. + Iterable, with manual or automatic offset commits, cooperative rebalance, + pluggable deserializers, and transactional-read isolation. +- :class:`~kafka.KafkaProducer` - high-level, asynchronous message producer. + Batches records into a background sender thread, with optional + idempotence, transactions, compression, and pluggable serializers. +- :class:`~kafka.KafkaAdminClient` - admin operations: topic, ACL, config, + consumer group, partition, quota, log-directory, and quorum management. + +.. toctree:: + :maxdepth: 1 + :hidden: + + KafkaConsumer + KafkaProducer + KafkaAdminClient + + +kafka.net +========= + +The clients share a single async networking layer (``kafka.net``). These +classes are exposed for advanced use cases - embedding the connection +pool, building a custom client on top of the kafka.net event loop, or +driving the protocol layer directly from the REPL. + +- :mod:`~kafka.net.manager` - connection pool and + high-level facade over the shared IO event loop. Each top-level client + owns one. +- :mod:`~kafka.net.connection` - per-broker async + connection: state machine, request/response correlation, and SASL + handshake. +- :mod:`~kafka.net.transport` - Async socket I/O with write buffering, + pause/resume hooks, and the asyncio-shaped protocol callback surface. +- :mod:`~kafka.net.inet` - DNS lookup + non-blocking connect, plus a + URL-scheme registry that resolves ``proxy_url`` to socket factories. +- :mod:`~kafka.net.http_connect` - Tunnels broker connections through + an HTTP CONNECT proxy (RFC 7231). +- :mod:`~kafka.net.socks5` - SOCKS5 client with optional username/password + authentication. + +.. toctree:: + :maxdepth: 1 + :hidden: + + manager + connection + transport + inet + http_connect + socks5 + + +other / misc +============ + +Lightweight data types used throughout the client APIs (and useful when +working with the lower-level protocol layer). + +- :class:`~kafka.cluster.ClusterMetadata` - in-memory cache of brokers, + topics, partitions, and the active controller. Refreshes itself on the + shared IO thread. +- :class:`~kafka.TopicPartition` - namedtuple identifying a partition as + ``(topic, partition)``. +- :class:`~kafka.OffsetAndMetadata` - committed-offset record + ``(offset, metadata, leader_epoch)``. +- :class:`~kafka.OffsetSpec` - enum for partition offset queries. +- :class:`~kafka.IsolationLevel` - enum for transactional isolation. +- :class:`~kafka.Serializer` - base class for serialization / deserialization + of key and value bytes. Includes helper classes `~kafka.DefaultSerializer` + and `~kafka.JsonSerializer`. +- :class:`~kafka.ConsumerRebalanceListener` - base class for consumer + class to receive join/rebalance group hooks. Also includes async + interface `~kafka.AsyncConsumerRebalanceListener`. + .. toctree:: :maxdepth: 1 + :hidden: - KafkaConsumer - KafkaProducer - KafkaAdminClient - KafkaConnectionManager - KafkaConnection - ClusterMetadata - OffsetAndMetadata - TopicPartition + ClusterMetadata + TopicPartition + OffsetAndMetadata + OffsetSpec + IsolationLevel + Serializer + ConsumerRebalanceListener + Partitioner diff --git a/docs/apidoc/net/connection.rst b/docs/apidoc/net/connection.rst new file mode 100644 index 000000000..8cf255404 --- /dev/null +++ b/docs/apidoc/net/connection.rst @@ -0,0 +1,8 @@ +kafka.net.connection +==================== + +.. module:: kafka.net.connection + +.. autoclass:: kafka.net.connection.KafkaConnection + :members: + :undoc-members: diff --git a/docs/apidoc/net/http_connect.rst b/docs/apidoc/net/http_connect.rst new file mode 100644 index 000000000..b5d534fc2 --- /dev/null +++ b/docs/apidoc/net/http_connect.rst @@ -0,0 +1,8 @@ +kafka.net.http_connect +====================== + +.. module:: kafka.net.http_connect + +.. autoclass:: kafka.net.http_connect.HttpConnectProxy + :members: + :undoc-members: diff --git a/docs/apidoc/net/inet.rst b/docs/apidoc/net/inet.rst new file mode 100644 index 000000000..9c051645f --- /dev/null +++ b/docs/apidoc/net/inet.rst @@ -0,0 +1,10 @@ +kafka.net.inet +============== + +.. module:: kafka.net.inet + +.. autoclass:: kafka.net.inet.KafkaNetSocket + :members: + :undoc-members: + +.. autofunction:: kafka.net.inet.create_connection diff --git a/docs/apidoc/net/manager.rst b/docs/apidoc/net/manager.rst new file mode 100644 index 000000000..065d871a8 --- /dev/null +++ b/docs/apidoc/net/manager.rst @@ -0,0 +1,8 @@ +kafka.net.manager +================= + +.. module:: kafka.net.manager + +.. autoclass:: kafka.net.manager.KafkaConnectionManager + :members: + :undoc-members: diff --git a/docs/apidoc/net/socks5.rst b/docs/apidoc/net/socks5.rst new file mode 100644 index 000000000..756ce00b1 --- /dev/null +++ b/docs/apidoc/net/socks5.rst @@ -0,0 +1,8 @@ +kafka.net.socks5 +================ + +.. module:: kafka.net.socks5 + +.. autoclass:: kafka.net.socks5.Socks5Proxy + :members: + :undoc-members: diff --git a/docs/apidoc/net/transport.rst b/docs/apidoc/net/transport.rst new file mode 100644 index 000000000..a6a9c2c0c --- /dev/null +++ b/docs/apidoc/net/transport.rst @@ -0,0 +1,12 @@ +kafka.net.transport +=================== + +.. module:: kafka.net.transport + +.. autoclass:: kafka.net.transport.KafkaTCPTransport + :members: + :undoc-members: + +.. autoclass:: kafka.net.transport.KafkaSSLTransport + :members: + :undoc-members: diff --git a/docs/conf.py b/docs/conf.py index 9b7175412..668ed4a8b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -110,7 +110,9 @@ # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + 'navigation_depth': -1, +} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] diff --git a/kafka/admin/_partitions.py b/kafka/admin/_partitions.py index e15c247c1..7ae7e2865 100644 --- a/kafka/admin/_partitions.py +++ b/kafka/admin/_partitions.py @@ -510,7 +510,7 @@ def _list_partition_offsets_process_response(response): ) return results - async def _async_list_partition_offsets(self, topic_partition_specs, isolation_level='read_uncommitted', timeout_ms=None): + async def _async_list_partition_offsets(self, topic_partition_specs, isolation_level=IsolationLevel.READ_UNCOMMITTED, timeout_ms=None): isolation_level = IsolationLevel.build_from(isolation_level) timer = Timer(self._validate_timeout(timeout_ms)) backoff_secs = self.config['retry_backoff_ms'] / 1000 @@ -539,7 +539,7 @@ async def _async_list_partition_offsets(self, topic_partition_specs, isolation_l await self._net.sleep(min(backoff_secs, max(0.0, timer.timeout_secs or 0.0))) return results - def list_partition_offsets(self, topic_partition_specs, isolation_level='read_uncommitted', timeout_ms=None): + def list_partition_offsets(self, topic_partition_specs, isolation_level=IsolationLevel.READ_UNCOMMITTED, timeout_ms=None): """Look up offsets for the given partitions by spec. Partitions are routed to their respective leader brokers via cluster @@ -554,9 +554,8 @@ def list_partition_offsets(self, topic_partition_specs, isolation_level='read_un wire-level sentinel). Keyword Arguments: - isolation_level (str, optional): One of ``'read_uncommitted'`` - (default) or ``'read_committed'``. ``read_committed`` requires - broker support for ListOffsets v2+. + isolation_level (IsolationLevel, optional): Requires broker support + for ListOffsets v2+. Default: IsolationLevel.READ_UNCOMMITTED. timeout_ms (int, optional): Maximum time to spend retrying NotLeaderForPartitionError. Default: ``request_timeout_ms``. diff --git a/kafka/admin/client.py b/kafka/admin/client.py index b7ecf2b7f..841060b6e 100644 --- a/kafka/admin/client.py +++ b/kafka/admin/client.py @@ -37,20 +37,6 @@ class KafkaAdminClient( ): """A class for administering the Kafka cluster. - Warning: - This is an unstable interface that was recently added and is subject to - change without warning. In particular, many methods currently return - raw protocol tuples. In future releases, we plan to make these into - nicer, more pythonic objects. Unfortunately, this will likely break - those interfaces. - - The KafkaAdminClient class will negotiate for the latest version of each message - protocol format supported by both the kafka-python client library and the - Kafka broker. Usage of optional fields from protocol versions that are not - supported by the broker will result in IncompatibleBrokerVersion exceptions. - - Use of this class requires a minimum broker version >= 0.10.0.0. - Keyword Arguments: bootstrap_servers: 'host[:port]' string (or list of 'host[:port]' strings) that the consumer should contact to bootstrap initial @@ -137,6 +123,7 @@ class KafkaAdminClient( Examples:: (4, 3) most recent broker release, enable all supported features + (2, 7) support SCRAM user credential apis (0, 11) enables message format v2 (internal) (0, 10, 0) enables sasl authentication and message format v1 (0, 9) enables full group coordination features with automatic