Embed Size (px)
Citation preview
MANCHESTER LONDON NEW YORKCC BY-NC 3.0
CC BY-NC 3.0
Screenshot
CC BY-NC 3.0
Before: the mess
CC BY-NC 3.0
Before: the mess
CC BY-NC 3.0
Before: the mess
Cassandra
Akka & Spray C++ & CUDARabbitMQ
CC BY-NC 3.0
Before: the mess
scene topic
identity topic
(Map[String, String], Array[Byte])
(Map[String, String], Array[Byte])
CC BY-NC 3.0
Before: the mess
_tmp_x topics
JSON
scene topic
identity topic
(Map[String, String], Array[Byte])
(Map[String, String], Array[Byte])
CC BY-NC 3.0
Before: the mess
fire-and-forget at-most-once
non-durablefire-and-forgetat-most-once
CC BY-NC 3.0
Asynchronous request–response orchestrationclass Orchestrator extends Actor with ActorFSM[Orchestrator.State, Orchestrator.Data] {
startWith(Idle, UninitializedData) when(Idle, idleTimeout)(idleSF) when(ImageProcessing, stepTimeout)(imageProcessingSF) when(WaitingForProcessingResult, stepTimeout)(waitingForProcessingSF) whenUnhandled(timeoutSF)
onTransition { case _ -> Aborted => ??? ... }
def idleSF: StateFunction = ??? def imageProcessingSF: StateFunction = ??? def waitingForProcessingSF: StateFunction = ???
def timeoutSF: StateFunction = { case Event(StateTimeout, data: RunningTransactionData) => goto(Aborted) }
}
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
Akka
Cassandra
Kafka
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
tweet-image topic
identity group
scene group
scene topic
identity topic
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
tweet-image topic
scene topic
identity topic
message Scene { … }
message Identity { … }
bytes image;
message Envelope { int32 version = 1; int64 processingTimestamp = 2; int64 ingestionTimestamp = 3; string correlationId = 4; string messageId = 5; string messageType = 6; bytes payload = 7;}
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
at-least-once I
at-least-once II
at-most-oncefire-and-forget
CC BY-NC 3.0
Persistence and formatsmessage Envelope { int32 version = 1; int64 processingTimestamp = 2; int64 ingestionTimestamp = 3; string correlationId = 4; string messageId = 5; string messageType = 6; bytes payload = 7;}
CC BY-NC 3.0
Persistence and formatsmessage Scene { message Label { string label = 1; double score = 2; } repeated Label labels = 3;}
CC BY-NC 3.0
Persistence and formatsmessage Identity { oneof face { IdentifiedFace identifiedFace = 1; UnknownFace unknownFace = 2; } message IdentifiedFace { string name = 1; double score = 2; } message UnknownFace { }}
CC BY-NC 3.0
Persistence and formatsmessage IdentifyFace { int64 ingestionTimestamp = 2; string correlationId = 3; string handle = 4; bytes image = 5;}message IdentifyFaces { repeated IdentifyFace identifyFaces = 1;}message FaceImage { double confidence = 1; int32 x = 2; int32 y = 3; int32 w = 4; int32 h = 5; bytes rgbBitmap = 6;}
CC BY-NC 3.0
Fire–and–forget sendobject Act { def props(config: Config): Props = { val producerConf = KafkaProducer.Conf(config.getConfig("..."), new StringSerializer, KafkaSerializer[Envelope](_.toByteArray)) Props(classOf[Act], producerConf) } }
class Act(producerConf: KafkaProducer.Conf[String, Envelope]) extends Actor { private[this] val producer = KafkaProducer(conf = producerConf)
override def receive: Receive = { case TweetImage(handle, content) => producer.send(KafkaProducerRecord("tweet-image", handle, Envelope(version = 100, ingestionTimestamp = System.nanoTime(), processingTimestamp = System.nanoTime(), messageId = UUID.randomUUID().toString, correlationId = UUID.randomUUID().toString, payload = content))) } }
CC BY-NC 3.0
At least once delivery Iclass SceneClassifierActor(…) extends Actor { private[this] val kafkaConsumerActor = context.actorOf(…) private[this] val producer = KafkaProducer(…)
override def receive: Receive = { case extractor(consumerRecords) => val futures = consumerRecords.pairs.flatMap { case (Some(handle), envelope) => val outEnvelope = … Some(producer.send(KafkaProducerRecord("scene", handle, outEnvelope))) } import context.dispatcher Future.sequence(futures).onSuccess { case _ => kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true) } }
}
CC BY-NC 3.0
At least once delivery IIclass IdentityMatcherActor(...) extends PersistentActor with AtLeastOnceDelivery { override val persistenceId: String = "identity-matcher-actor"
def identifyFacesAndSend(identifyFaces: Seq[IdentifyFace])(implicit executor: ExecutionContext): Future[Unit] = { // Future.sequence(producer.send(...)) }
def handleIdentifyFace: Receive = { case (deliveryId: Long, identifyFaces: IdentifyFaces) => import context.dispatcher identifyFacesAndSend(identifyFaces.identifyFaces).onSuccess { case _ => confirmDelivery(deliveryId) } case IdentifyFaces(faces) => import context.dispatcher identifyFacesAndSend(faces).onFailure { case _ => self ! Kill } }
override def receiveRecover: Receive = handleIdentifyFace
override def receiveCommand: Receive = handleIdentifyFace orElse { case extractor(consumerRecords) => val identifyFaces = consumerRecords.pairs.flatMap { case (Some(handle), envelope) => Some(IdentifyFace(envelope.ingestionTimestamp, envelope.correlationId, handle, envelope.payload)) } persist(IdentifyFaces(identifyFaces = identifyFaces)) { result => deliver(self.path)(deliveryId => (deliveryId, result)) sender() ! Confirm(consumerRecords.offsets, commit = true) } } }
CC BY-NC 3.0
At most once deliveryclass DashboardSinkActor(...) extends Actor { private[this] val kafkaConsumerActor = context.actorOf(...)
override def receive: Receive = { case extractor(consumerRecords) => consumerRecords.pairs.foreach { case (None, _) => case (Some(handle), envelope) => context.system.eventStream.publish( TweetEnvelope(version = 100, handle = handle, ingestionTimestamp = envelope.ingestionTimestamp, messageId = envelope.messageId, messageType = envelope.messageType, payload = envelope.payload)) }
kafkaConsumerActor ! Confirm(consumerRecords.offsets, commit = true) }
}
CC BY-NC 3.0
ENOUGH SLIDES!
CC BY-NC 3.0
After: proper microservices
scene
identity
ingest dashboard
MANCHESTER LONDON NEW YORKCC BY-NC 3.0
github.com/eigengo/reactive-summit-2016rsa16-models.s3-website-eu-west-1.amazonaws.com/{identity,scene}/{config,labels,params}
