blankensteiner commented on a change in pull request #71:
URL: https://github.com/apache/pulsar-dotpulsar/pull/71#discussion_r654686078
##########
File path: samples/Producing/Program.cs
##########
@@ -50,7 +50,7 @@ private static async Task Main()
private static async Task ProduceMessages(IProducer<string> producer,
CancellationToken cancellationToken)
{
var delay = TimeSpan.FromSeconds(5);
-
+ await producer.StateChangedTo(ProducerState.Connected,
cancellationToken: cancellationToken).ConfigureAwait(false);
Review comment:
This can be removed since you should never wait for a state before using
the producer
##########
File path: src/DotPulsar/Internal/Producer.cs
##########
@@ -1,159 +1,122 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-namespace DotPulsar.Internal
+namespace DotPulsar.Internal
{
using Abstractions;
using DotPulsar.Abstractions;
- using DotPulsar.Exceptions;
- using DotPulsar.Internal.Extensions;
using Events;
- using Microsoft.Extensions.ObjectPool;
+ using Exceptions;
using System;
- using System.Buffers;
+ using System.Collections.Concurrent;
using System.Threading;
using System.Threading.Tasks;
- public sealed class Producer<TMessage> : IEstablishNewChannel,
IProducer<TMessage>
+ public sealed class Producer<TMessage> : IProducer<TMessage>
{
- private readonly ObjectPool<PulsarApi.MessageMetadata>
_messageMetadataPool;
private readonly Guid _correlationId;
private readonly IRegisterEvent _eventRegister;
- private IProducerChannel _channel;
private readonly IExecute _executor;
private readonly IStateChanged<ProducerState> _state;
- private readonly IProducerChannelFactory _factory;
- private readonly ISchema<TMessage> _schema;
- private readonly SequenceId _sequenceId;
+ private readonly PulsarClient _pulsarClient;
+ private readonly ProducerOptions<TMessage> _options;
+ private readonly ConcurrentDictionary<int, IProducer<TMessage>>
_producers;
+ private readonly IMessageRouter _messageRouter;
+ private readonly CancellationTokenSource _cts = new();
+ private int _producersCount;
private int _isDisposed;
-
public Uri ServiceUrl { get; }
public string Topic { get; }
public Producer(
Guid correlationId,
Uri serviceUrl,
string topic,
- ulong initialSequenceId,
IRegisterEvent registerEvent,
- IProducerChannel initialChannel,
IExecute executor,
IStateChanged<ProducerState> state,
- IProducerChannelFactory factory,
- ISchema<TMessage> schema)
+ ProducerOptions<TMessage> options,
+ PulsarClient pulsarClient
+ )
{
- var messageMetadataPolicy = new
DefaultPooledObjectPolicy<PulsarApi.MessageMetadata>();
- _messageMetadataPool = new
DefaultObjectPool<PulsarApi.MessageMetadata>(messageMetadataPolicy);
_correlationId = correlationId;
ServiceUrl = serviceUrl;
Topic = topic;
- _sequenceId = new SequenceId(initialSequenceId);
_eventRegister = registerEvent;
- _channel = initialChannel;
_executor = executor;
_state = state;
- _factory = factory;
- _schema = schema;
_isDisposed = 0;
+ _options = options;
+ _pulsarClient = pulsarClient;
+ _messageRouter = options.MessageRouter;
+
+ _producers = new ConcurrentDictionary<int, IProducer<TMessage>>(1,
31);
- _eventRegister.Register(new ProducerCreated(_correlationId));
+ UpdatePartitions(_cts.Token);
}
- public async ValueTask<ProducerState> OnStateChangeTo(ProducerState
state, CancellationToken cancellationToken)
- => await _state.StateChangedTo(state,
cancellationToken).ConfigureAwait(false);
+ private void CreateSubProducers(int startIndex, int count)
+ {
+ if (count == 0)
+ {
+ var producer = _pulsarClient.NewSubProducer(Topic, _options,
_executor, _correlationId);
+ _producers[0] = producer;
+ return;
+ }
- public async ValueTask<ProducerState> OnStateChangeFrom(ProducerState
state, CancellationToken cancellationToken)
- => await _state.StateChangedFrom(state,
cancellationToken).ConfigureAwait(false);
+ for (var i = startIndex; i < count; ++i)
+ {
+ var producer = _pulsarClient.NewSubProducer(Topic, _options,
_executor, _correlationId, (uint) i);
+ _producers[i] = producer;
+ }
+ }
+
+ private async void UpdatePartitions(CancellationToken
cancellationToken)
Review comment:
"async void" is a no-go, because it is a "fire and forget". I think an
exception here will crash the application.
We need to make this call part of the ongoing connect/reconnect feature
(meaning the process/management).
##########
File path: tests/DotPulsar.Tests/Internal/PartitionedProducerProcessTests.cs
##########
@@ -0,0 +1,115 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace DotPulsar.Tests.Internal
+{
+ using Abstractions;
+ using DotPulsar.Internal;
+ using DotPulsar.Internal.Abstractions;
+ using DotPulsar.Internal.Events;
+ using NSubstitute;
+ using System;
+ using System.Collections.Concurrent;
+ using System.Collections.Generic;
+ using System.Threading.Tasks;
+ using Xunit;
+
+ public class PartitionedProducerProcessTests
+ {
+ [Fact]
+ public Task
TestPartitionedProducerStateManage_WhenSubProducersStateChange_ThenPartitionedProducerStateChangeCorrectly()
Review comment:
When not using async/await, you can just have the test return void.
##########
File path: src/DotPulsar/Internal/Producer.cs
##########
@@ -1,159 +1,122 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-namespace DotPulsar.Internal
+namespace DotPulsar.Internal
Review comment:
We need the apache license header here
##########
File path: tests/DotPulsar.Tests/Internal/PartitionedProducerProcessTests.cs
##########
@@ -0,0 +1,115 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace DotPulsar.Tests.Internal
+{
+ using Abstractions;
+ using DotPulsar.Internal;
+ using DotPulsar.Internal.Abstractions;
+ using DotPulsar.Internal.Events;
+ using NSubstitute;
+ using System;
+ using System.Collections.Concurrent;
+ using System.Collections.Generic;
+ using System.Threading.Tasks;
+ using Xunit;
+
+ public class PartitionedProducerProcessTests
+ {
+ [Fact]
+ public Task
TestPartitionedProducerStateManage_WhenSubProducersStateChange_ThenPartitionedProducerStateChangeCorrectly()
+ {
+ var connectionPool = Substitute.For<IConnectionPool>();
+ var establishNewChannel = Substitute.For<IEstablishNewChannel>();
+ var producer = Substitute.For<IProducer>();
+
+ var processManager = new ProcessManager(connectionPool);
+
+ var producerGuids = new Dictionary<uint, Guid>(3);
+ var producersGroup = new ConcurrentDictionary<uint,
IProducer>(Environment.ProcessorCount, 3);
+ var partitionedProducerGuid = Guid.NewGuid();
+
+ for (uint i = 0; i < 3; i++)
+ {
+ var stateManager = new
StateManager<ProducerState>(ProducerState.Disconnected, ProducerState.Closed,
ProducerState.Faulted);
+ var correlationId = Guid.NewGuid();
+ var process = new ProducerProcess(correlationId, stateManager,
establishNewChannel, processManager, partitionedProducerGuid);
+ producerGuids[i] = correlationId;
+ producersGroup[i] = producer;
+ processManager.Add(process);
+ }
+
+ var partitionedStateManager =
+ new StateManager<ProducerState>(ProducerState.Disconnected,
ProducerState.Closed, ProducerState.Faulted);
+
+ var producerProcess = new ProducerProcess(partitionedProducerGuid,
partitionedStateManager, null, new ProcessManager(connectionPool));
+ processManager.Add(producerProcess);
+ processManager.Register(new
UpdatePartitions(partitionedProducerGuid, (uint) producersGroup.Count));
+
+ // Test initial channel
+ processManager.Register(new ChannelDisconnected(producerGuids[0]));
+ Assert.Equal(ProducerState.Disconnected,
partitionedStateManager.CurrentState);
+ processManager.Register(new ChannelDisconnected(producerGuids[1]));
+ Assert.Equal(ProducerState.Disconnected,
partitionedStateManager.CurrentState);
+ processManager.Register(new ChannelDisconnected(producerGuids[2]));
+ Assert.Equal(ProducerState.Disconnected,
partitionedStateManager.CurrentState);
+
+ // Test connect
+ Assert.Equal(ProducerState.Disconnected,
partitionedStateManager.CurrentState);
+ processManager.Register(new ChannelConnected(producerGuids[0]));
+ Assert.Equal(ProducerState.PartiallyConnected,
partitionedStateManager.CurrentState);
+ processManager.Register(new ChannelConnected(producerGuids[1]));
+ Assert.Equal(ProducerState.PartiallyConnected,
partitionedStateManager.CurrentState);
+ processManager.Register(new ChannelConnected(producerGuids[2]));
+ Assert.Equal(ProducerState.Connected,
partitionedStateManager.CurrentState);
+
+ // Test disconnect
+ processManager.Register(new ChannelDisconnected(producerGuids[1]));
+ Assert.Equal(ProducerState.PartiallyConnected,
partitionedStateManager.CurrentState);
+
+ // Test reconnect
+ processManager.Register(new ChannelConnected(producerGuids[1]));
+ Assert.Equal(ProducerState.Connected,
partitionedStateManager.CurrentState);
+
+ // Test fault
+ processManager.Register(new ExecutorFaulted(producerGuids[1]));
+ Assert.Equal(ProducerState.Faulted,
partitionedStateManager.CurrentState);
+
+ return Task.CompletedTask;
+ }
+
+ [Fact]
+ public Task
TestUpdatePartitions_WhenIncreasePartitions_ThenPartitionedProducerStateChangeCorrectly()
Review comment:
When not using async/await, you can just have the test return void.
##########
File path: src/DotPulsar/Internal/ProducerProcess.cs
##########
@@ -15,52 +15,141 @@
namespace DotPulsar.Internal
{
using Abstractions;
+ using Events;
using System;
+ using System.Threading;
using System.Threading.Tasks;
public sealed class ProducerProcess : Process
{
private readonly IStateManager<ProducerState> _stateManager;
- private readonly IEstablishNewChannel _producer;
+ private readonly IEstablishNewChannel? _producer;
+
+ // The following variables are only used when this is the process for
parent producer.
+ private readonly IRegisterEvent _processManager;
+ private int _partitionsCount;
+ private int _connectedProducersCount;
+ private int _initialProducersCount;
+
+ // The following variables are only used for sub producer
+ private readonly Guid? _partitionedProducerId;
public ProducerProcess(
Guid correlationId,
IStateManager<ProducerState> stateManager,
- IEstablishNewChannel producer) : base(correlationId)
+ IEstablishNewChannel? producer,
+ IRegisterEvent processManager,
+ Guid? partitionedProducerId = null) : base(correlationId)
{
_stateManager = stateManager;
_producer = producer;
+ _processManager = processManager;
+ _partitionedProducerId = partitionedProducerId;
}
public override async ValueTask DisposeAsync()
{
- _stateManager.SetState(ProducerState.Closed);
+ SetState(ProducerState.Closed);
CancellationTokenSource.Cancel();
- await _producer.DisposeAsync().ConfigureAwait(false);
+
+ if (_producer != null)
+ await _producer.DisposeAsync().ConfigureAwait(false);
+ }
+
+ protected override void HandleExtend(IEvent e)
Review comment:
I think we can do this a bit more simply. The SubProducers have their
own processes and own states, so we just need to monitor them (like the
end-user does with the producer). When one of the SubProducers changes state,
we just need to look at the "total states" and then, if needed, update the
PartitionedProducer's state (the one the end-user is monitoring).
The component monitoring the state changes of the SubProducers could also
initially and periodically check what the total number of partitions is for the
topic and therefore also create and delete the SubProducers.
I'm probably not being very clear now, so should I have it a try?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
