RobertIndie commented on a change in pull request #71: URL: https://github.com/apache/pulsar-dotpulsar/pull/71#discussion_r655121619
########## File path: src/DotPulsar/Internal/Producer.cs ########## @@ -1,159 +1,122 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -namespace DotPulsar.Internal +namespace DotPulsar.Internal Review comment: Sorry for the mistake. :joy: ########## File path: src/DotPulsar/Internal/Producer.cs ########## @@ -1,159 +1,122 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -namespace DotPulsar.Internal +namespace DotPulsar.Internal { using Abstractions; using DotPulsar.Abstractions; - using DotPulsar.Exceptions; - using DotPulsar.Internal.Extensions; using Events; - using Microsoft.Extensions.ObjectPool; + using Exceptions; using System; - using System.Buffers; + using System.Collections.Concurrent; using System.Threading; using System.Threading.Tasks; - public sealed class Producer<TMessage> : IEstablishNewChannel, IProducer<TMessage> + public sealed class Producer<TMessage> : IProducer<TMessage> { - private readonly ObjectPool<PulsarApi.MessageMetadata> _messageMetadataPool; private readonly Guid _correlationId; private readonly IRegisterEvent _eventRegister; - private IProducerChannel _channel; private readonly IExecute _executor; private readonly IStateChanged<ProducerState> _state; - private readonly IProducerChannelFactory _factory; - private readonly ISchema<TMessage> _schema; - private readonly SequenceId _sequenceId; + private readonly PulsarClient _pulsarClient; + private readonly ProducerOptions<TMessage> _options; + private readonly ConcurrentDictionary<int, IProducer<TMessage>> _producers; + private readonly IMessageRouter _messageRouter; + private readonly CancellationTokenSource _cts = new(); + private int _producersCount; private int _isDisposed; - public Uri ServiceUrl { get; } public string Topic { get; } public Producer( Guid correlationId, Uri serviceUrl, string topic, - ulong initialSequenceId, IRegisterEvent registerEvent, - IProducerChannel initialChannel, IExecute executor, IStateChanged<ProducerState> state, - IProducerChannelFactory factory, - ISchema<TMessage> schema) + ProducerOptions<TMessage> options, + PulsarClient pulsarClient + ) { - var messageMetadataPolicy = new DefaultPooledObjectPolicy<PulsarApi.MessageMetadata>(); - _messageMetadataPool = new DefaultObjectPool<PulsarApi.MessageMetadata>(messageMetadataPolicy); _correlationId = correlationId; ServiceUrl = serviceUrl; Topic = topic; - _sequenceId = new SequenceId(initialSequenceId); _eventRegister = registerEvent; - _channel = initialChannel; _executor = executor; _state = state; - _factory = factory; - _schema = schema; _isDisposed = 0; + _options = options; + _pulsarClient = pulsarClient; + _messageRouter = options.MessageRouter; + + _producers = new ConcurrentDictionary<int, IProducer<TMessage>>(1, 31); - _eventRegister.Register(new ProducerCreated(_correlationId)); + UpdatePartitions(_cts.Token); } - public async ValueTask<ProducerState> OnStateChangeTo(ProducerState state, CancellationToken cancellationToken) - => await _state.StateChangedTo(state, cancellationToken).ConfigureAwait(false); + private void CreateSubProducers(int startIndex, int count) + { + if (count == 0) + { + var producer = _pulsarClient.NewSubProducer(Topic, _options, _executor, _correlationId); + _producers[0] = producer; + return; + } - public async ValueTask<ProducerState> OnStateChangeFrom(ProducerState state, CancellationToken cancellationToken) - => await _state.StateChangedFrom(state, cancellationToken).ConfigureAwait(false); + for (var i = startIndex; i < count; ++i) + { + var producer = _pulsarClient.NewSubProducer(Topic, _options, _executor, _correlationId, (uint) i); + _producers[i] = producer; + } + } + + private async void UpdatePartitions(CancellationToken cancellationToken) Review comment: I have moved it to `EstablishNewChannel `. However, there are two types of logic in `EstablishNewChannel` currently: `Create new channel` and `Update partitions`. Can we rename `EstablishNewChannel` to `InitialConnection`? ########## File path: src/DotPulsar/Internal/ProducerProcess.cs ########## @@ -15,52 +15,141 @@ namespace DotPulsar.Internal { using Abstractions; + using Events; using System; + using System.Threading; using System.Threading.Tasks; public sealed class ProducerProcess : Process { private readonly IStateManager<ProducerState> _stateManager; - private readonly IEstablishNewChannel _producer; + private readonly IEstablishNewChannel? _producer; + + // The following variables are only used when this is the process for parent producer. + private readonly IRegisterEvent _processManager; + private int _partitionsCount; + private int _connectedProducersCount; + private int _initialProducersCount; + + // The following variables are only used for sub producer + private readonly Guid? _partitionedProducerId; public ProducerProcess( Guid correlationId, IStateManager<ProducerState> stateManager, - IEstablishNewChannel producer) : base(correlationId) + IEstablishNewChannel? producer, + IRegisterEvent processManager, + Guid? partitionedProducerId = null) : base(correlationId) { _stateManager = stateManager; _producer = producer; + _processManager = processManager; + _partitionedProducerId = partitionedProducerId; } public override async ValueTask DisposeAsync() { - _stateManager.SetState(ProducerState.Closed); + SetState(ProducerState.Closed); CancellationTokenSource.Cancel(); - await _producer.DisposeAsync().ConfigureAwait(false); + + if (_producer != null) + await _producer.DisposeAsync().ConfigureAwait(false); + } + + protected override void HandleExtend(IEvent e) Review comment: What confuses me is the UpdatePartitions part. I think Producer class need to be responsible for sub producers creation. The producer process does not create or delete sub producers. If we want to implement in this way, we may need to share the collection of sub producers between Producer and ParoducerProcess. But it seems more complex. ########## File path: src/DotPulsar/SinglePartitionRouter.cs ########## @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace DotPulsar +{ + using Abstractions; + using HashDepot; + using System; + using System.Text; + + /// <summary> + /// If no key is provided, the producer will randomly pick one single partition and publish all the messages + /// into that partition. While if a key is specified on the message, the partitioned producer will hash the + /// key and assign message to a particular partition. + /// </summary> + public sealed class SinglePartitionRouter : IMessageRouter + { + private int? _partitionIndex; + + internal SinglePartitionRouter(int? partitionIndex = null) + { + _partitionIndex = partitionIndex; + } + + /// <summary> + /// Choose a partition in single partition routing mode + /// </summary> + public int ChoosePartition(MessageMetadata? messageMetadata, int partitionsCount) + { + if (messageMetadata != null && !string.IsNullOrEmpty(messageMetadata.Key)) + { + return (int) MurmurHash3.Hash32(Encoding.UTF8.GetBytes(messageMetadata.Key ?? string.Empty), 0) % partitionsCount; + } + + _partitionIndex ??= new Random().Next(0, partitionsCount); Review comment: This implementation is the same as the java client and it makes sense. SinglePartitionRouter simply selects a random partition when creating the producer, and all subsequent messages(without key assigned) are sent to that partition. Currently, pulsar does not support reducing partitions. The selected partition should not be changed when partition expansion occurs -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
