20240409 rework orca markov (#2699)

Improve the models and what can be performed in the orca benchmarks.

---------

Co-authored-by: Sebastiano Tocci <seba.tocci@gmail.com>
Co-authored-by: Sebastiano Tocci <sebastiano.tocci@proton.me>
This commit is contained in:
Firstyear 2024-04-17 09:35:16 +10:00 committed by GitHub
parent d7834b52e6
commit 62bbd7e3ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 432 additions and 94 deletions

View file

@ -176,12 +176,6 @@ opentelemetry-otlp = { version = "0.13.0", default-features = false, features =
"grpc-tonic",
] }
opentelemetry_sdk = "0.20.0"
opentelemetry-stdout = { version = "0.1.0", features = [
"logs",
"metrics",
"trace",
] }
tonic = "0.10.2"
tracing-opentelemetry = "0.21.0"
paste = "^1.0.14"

View file

@ -1,6 +1,6 @@
# Orca - A Kanidm Load Testing Tool
Make a profile
Make a profile.toml
```shell
orca setup-wizard --idm-admin-password ... \

View file

@ -5,7 +5,7 @@ pub enum Error {
KanidmClient,
ProfileBuilder,
Tokio,
Interupt,
Interrupt,
Crossbeam,
InvalidState,
}

View file

@ -1,10 +1,11 @@
use crate::error::Error;
use crate::kani::KanidmOrcaClient;
use crate::model::ActorRole;
use crate::profile::Profile;
use crate::state::{Credential, Flag, Model, Person, PreflightState, State};
use rand::distributions::{Alphanumeric, DistString};
use rand::seq::SliceRandom;
use rand::SeedableRng;
use crate::state::{Credential, Flag, Group, Model, Person, PreflightState, State};
use rand::distributions::{Alphanumeric, DistString, Uniform};
use rand::seq::{index, SliceRandom};
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha8Rng;
use std::collections::BTreeSet;
@ -53,9 +54,23 @@ pub async fn populate(_client: &KanidmOrcaClient, profile: Profile) -> Result<St
let preflight_flags = vec![Flag::DisableAllPersonsMFAPolicy];
// PHASE 1 - generate a pool of persons that are not-yet created for future import.
// todo! may need a random username vec for later stuff
// PHASE 2 - generate persons
// PHASE 2 - generate groups for integration access, assign roles to groups.
// These decide what each person is supposed to do with their life.
let mut groups = vec![
Group {
name: "role_people_pii_reader".to_string(),
role: ActorRole::PeoplePiiReader,
..Default::default()
},
Group {
name: "role_people_self_write_mail".to_string(),
role: ActorRole::PeopleSelfWriteMail,
..Default::default()
},
];
// PHASE 3 - generate persons
// - assign them credentials of various types.
let mut persons = Vec::with_capacity(profile.person_count() as usize);
let mut person_usernames = BTreeSet::new();
@ -88,17 +103,17 @@ pub async fn populate(_client: &KanidmOrcaClient, profile: Profile) -> Result<St
let password = random_password(&mut seeded_rng);
// TODO: Add more and different "models" to each person for their actions.
let roles = BTreeSet::new();
let model = Model::Basic;
// =======
// Data is ready, make changes to the server. These should be idempotent if possible.
let p = Person {
preflight_state: PreflightState::Present,
username: username.clone(),
display_name,
member_of: BTreeSet::default(),
roles,
credential: Credential::Password { plain: password },
model,
};
@ -109,7 +124,43 @@ pub async fn populate(_client: &KanidmOrcaClient, profile: Profile) -> Result<St
persons.push(p);
}
// PHASE 3 - generate groups for integration access, assign persons.
// Now, assign persons to roles.
//
// We do this by iterating through our roles, and then assigning
// them a baseline of required accounts with some variation. This
// way in each test it's guaranteed that *at least* one person
// to each role always will exist and be operational.
for group in groups.iter_mut() {
// For now, our baseline is 50%. We can adjust this in future per
// role for example.
let baseline = persons.len() / 2;
let inverse = persons.len() - baseline;
// Randomly add extra from the inverse
let extra = Uniform::new(0, inverse);
let persons_to_choose = baseline + seeded_rng.sample(extra);
assert!(persons_to_choose <= persons.len());
debug!(?persons_to_choose);
let person_index = index::sample(&mut seeded_rng, persons.len(), persons_to_choose);
// Order doesn't matter, lets optimise for linear lookup.
let mut person_index = person_index.into_vec();
person_index.sort_unstable();
for p_idx in person_index {
let person = persons.get_mut(p_idx).unwrap();
// Add the person to the group.
group.members.insert(person.username.clone());
// Add the reverse links, this allows the person in the test
// to know their roles
person.roles.insert(group.role.clone());
}
}
// PHASE 4 - generate groups for user modification rights
@ -117,13 +168,14 @@ pub async fn populate(_client: &KanidmOrcaClient, profile: Profile) -> Result<St
// PHASE 6 - generate integrations -
// PHASE 7 - given the intergariotns and groupings,
// PHASE 7 - given the integrations and groupings,
// Return the state.
let state = State {
profile,
// ---------------
groups,
preflight_flags,
persons,
};

View file

@ -84,7 +84,7 @@ impl KanidmOrcaClient {
})
}
pub async fn person_set_pirmary_password_only(
pub async fn person_set_primary_password_only(
&self,
username: &str,
password: &str,
@ -97,4 +97,45 @@ impl KanidmOrcaClient {
Error::KanidmClient
})
}
pub async fn group_set_members(&self, group_name: &str, members: &[&str]) -> Result<(), Error> {
self.idm_admin_client
.idm_group_set_members(group_name, members)
.await
.map_err(|err| {
error!(?err, ?group_name, "Unable to set group members");
Error::KanidmClient
})
}
pub async fn group_add_members(&self, group_name: &str, members: &[&str]) -> Result<(), Error> {
self.idm_admin_client
.idm_group_add_members(group_name, members)
.await
.map_err(|err| {
error!(?err, ?group_name, "Unable to add group members");
Error::KanidmClient
})
}
pub async fn group_exists(&self, group_name: &str) -> Result<bool, Error> {
self.idm_admin_client
.idm_group_get(group_name)
.await
.map(|e| e.is_some())
.map_err(|err| {
error!(?err, ?group_name, "Unable to check group");
Error::KanidmClient
})
}
pub async fn group_create(&self, group_name: &str) -> Result<(), Error> {
self.idm_admin_client
.idm_group_create(group_name, Some("idm_admins"))
.await
.map_err(|err| {
error!(?err, ?group_name, "Unable to create group");
Error::KanidmClient
})
}
}

View file

@ -1,4 +1,4 @@
// #![deny(warnings)]
#![deny(warnings)]
#![warn(unused_extern_crates)]
#![allow(clippy::panic)]
#![deny(clippy::unreachable)]
@ -26,7 +26,7 @@ mod error;
mod generate;
mod kani;
mod model;
mod model_basic;
mod models;
mod populate;
mod profile;
mod run;

View file

@ -3,13 +3,16 @@ use crate::run::{EventDetail, EventRecord};
use crate::state::*;
use std::time::{Duration, Instant};
use kanidm_client::KanidmClient;
use kanidm_client::{ClientError, KanidmClient};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
pub enum TransitionAction {
Login,
Logout,
PrivilegeReauth,
WriteAttributePersonMail,
}
// Is this the right way? Should transitions/delay be part of the actor model? Should
@ -31,10 +34,28 @@ pub enum TransitionResult {
Ok,
// We need to re-authenticate, the session expired.
// AuthenticationNeeded,
// An error occured.
// An error occurred.
Error,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)]
pub enum ActorRole {
#[default]
None,
PeoplePiiReader,
PeopleSelfWriteMail,
}
impl ActorRole {
pub fn requires_membership_to(&self) -> Option<&[&str]> {
match self {
ActorRole::None => None,
ActorRole::PeoplePiiReader => Some(&["idm_people_pii_read"]),
ActorRole::PeopleSelfWriteMail => Some(&["idm_people_self_write_mail"]),
}
}
}
#[async_trait]
pub trait ActorModel {
async fn transition(
@ -57,31 +78,59 @@ pub async fn login(
.await
}
};
let end = Instant::now();
let duration = end.duration_since(start);
let duration = Instant::now().duration_since(start);
Ok(parse_call_result_into_transition_result_and_event_record(
result,
EventDetail::Login,
start,
duration,
))
}
match result {
Ok(_) => Ok((
TransitionResult::Ok,
EventRecord {
start,
duration,
details: EventDetail::Authentication,
},
)),
Err(client_err) => {
debug!(?client_err);
Ok((
TransitionResult::Error,
EventRecord {
start,
duration,
details: EventDetail::Error,
},
))
}
}
pub async fn person_set_self_mail(
client: &KanidmClient,
person: &Person,
values: &[&str],
) -> Result<(TransitionResult, EventRecord), Error> {
// Should we measure the time of each call rather than the time with multiple calls?
let person_username = person.username.as_str();
let start = Instant::now();
let result = client
.idm_person_account_set_attr(person_username, "mail", values)
.await;
let duration = Instant::now().duration_since(start);
let parsed_result = parse_call_result_into_transition_result_and_event_record(
result,
EventDetail::PersonSetSelfMail,
start,
duration,
);
Ok(parsed_result)
}
pub async fn privilege_reauth(
client: &KanidmClient,
person: &Person,
) -> Result<(TransitionResult, EventRecord), Error> {
let start = Instant::now();
let result = match &person.credential {
Credential::Password { plain } => client.reauth_simple_password(plain.as_str()).await,
};
let duration = Instant::now().duration_since(start);
let parsed_result = parse_call_result_into_transition_result_and_event_record(
result,
EventDetail::PersonReauth,
start,
duration,
);
Ok(parsed_result)
}
pub async fn logout(
@ -90,29 +139,41 @@ pub async fn logout(
) -> Result<(TransitionResult, EventRecord), Error> {
let start = Instant::now();
let result = client.logout().await;
let end = Instant::now();
let duration = Instant::now().duration_since(start);
let duration = end.duration_since(start);
Ok(parse_call_result_into_transition_result_and_event_record(
result,
EventDetail::Logout,
start,
duration,
))
}
fn parse_call_result_into_transition_result_and_event_record<T>(
result: Result<T, ClientError>,
details: EventDetail,
start: Instant,
duration: Duration,
) -> (TransitionResult, EventRecord) {
match result {
Ok(_) => Ok((
Ok(_) => (
TransitionResult::Ok,
EventRecord {
start,
duration,
details: EventDetail::Logout,
details,
},
)),
),
Err(client_err) => {
debug!(?client_err);
Ok((
(
TransitionResult::Error,
EventRecord {
start,
duration,
details: EventDetail::Error,
},
))
)
}
}
}

View file

@ -14,20 +14,20 @@ enum State {
Authenticated,
}
pub struct ActorBasic {
pub struct ActorAuthOnly {
state: State,
}
impl ActorBasic {
impl ActorAuthOnly {
pub fn new() -> Self {
ActorBasic {
ActorAuthOnly {
state: State::Unauthenticated,
}
}
}
#[async_trait]
impl ActorModel for ActorBasic {
impl ActorModel for ActorAuthOnly {
async fn transition(
&mut self,
client: &KanidmClient,
@ -43,16 +43,16 @@ impl ActorModel for ActorBasic {
let (result, event) = match transition.action {
TransitionAction::Login => model::login(client, person).await,
TransitionAction::Logout => model::logout(client, person).await,
_ => Err(Error::InvalidState),
}?;
// Given the result, make a choice about what text.
self.next_state(result);
self.next_state(transition.action, result);
Ok(event)
}
}
impl ActorBasic {
impl ActorAuthOnly {
fn next_transition(&mut self) -> Transition {
match self.state {
State::Unauthenticated => Transition {
@ -66,20 +66,19 @@ impl ActorBasic {
}
}
fn next_state(&mut self, result: TransitionResult) {
// Is this a design flaw? We probably need to know what the state was that we
// requested to move to?
match (&self.state, result) {
(State::Unauthenticated, TransitionResult::Ok) => {
fn next_state(&mut self, action: TransitionAction, result: TransitionResult) {
match (&self.state, action, result) {
(State::Unauthenticated, TransitionAction::Login, TransitionResult::Ok) => {
self.state = State::Authenticated;
}
(State::Unauthenticated, TransitionResult::Error) => {
(State::Authenticated, TransitionAction::Logout, TransitionResult::Ok) => {
self.state = State::Unauthenticated;
}
(State::Authenticated, TransitionResult::Ok) => {
self.state = State::Unauthenticated;
// Shouldn't be reachable?
(_, _, TransitionResult::Ok) => {
unreachable!();
}
(State::Authenticated, TransitionResult::Error) => {
(_, _, TransitionResult::Error) => {
self.state = State::Unauthenticated;
}
}

View file

@ -0,0 +1,117 @@
use crate::model::{self, ActorModel, ActorRole, Transition, TransitionAction, TransitionResult};
use crate::error::Error;
use crate::run::EventRecord;
use crate::state::*;
use kanidm_client::KanidmClient;
use async_trait::async_trait;
use std::collections::BTreeSet;
use std::time::Duration;
enum State {
Unauthenticated,
Authenticated,
AuthenticatedWithReauth,
}
pub struct ActorBasic {
state: State,
}
impl ActorBasic {
pub fn new() -> Self {
ActorBasic {
state: State::Unauthenticated,
}
}
}
#[async_trait]
impl ActorModel for ActorBasic {
async fn transition(
&mut self,
client: &KanidmClient,
person: &Person,
) -> Result<EventRecord, Error> {
let transition = self.next_transition(&person.roles);
if let Some(delay) = transition.delay {
tokio::time::sleep(delay).await;
}
// Once we get to here, we want the transition to go ahead.
let (result, event) = match transition.action {
TransitionAction::Login => model::login(client, person).await,
TransitionAction::Logout => model::logout(client, person).await,
TransitionAction::PrivilegeReauth => model::privilege_reauth(client, person).await,
TransitionAction::WriteAttributePersonMail => {
let mail = format!("{}@example.com", person.username);
let values = &[mail.as_str()];
model::person_set_self_mail(client, person, values).await
}
}?;
self.next_state(transition.action, result);
Ok(event)
}
}
impl ActorBasic {
fn next_transition(&mut self, roles: &BTreeSet<ActorRole>) -> Transition {
match self.state {
State::Unauthenticated => Transition {
delay: None,
action: TransitionAction::Login,
},
State::Authenticated => Transition {
delay: Some(Duration::from_millis(100)),
action: TransitionAction::PrivilegeReauth,
},
State::AuthenticatedWithReauth => {
if roles.contains(&ActorRole::PeopleSelfWriteMail) {
Transition {
delay: Some(Duration::from_millis(200)),
action: TransitionAction::WriteAttributePersonMail,
}
} else {
Transition {
delay: Some(Duration::from_secs(5)),
action: TransitionAction::Logout,
}
}
}
}
}
fn next_state(&mut self, action: TransitionAction, result: TransitionResult) {
// Is this a design flaw? We probably need to know what the state was that we
// requested to move to?
match (&self.state, action, result) {
(State::Unauthenticated, TransitionAction::Login, TransitionResult::Ok) => {
self.state = State::Authenticated;
}
(State::Authenticated, TransitionAction::PrivilegeReauth, TransitionResult::Ok) => {
self.state = State::AuthenticatedWithReauth;
}
(
State::AuthenticatedWithReauth,
TransitionAction::WriteAttributePersonMail,
TransitionResult::Ok,
) => {
self.state = State::AuthenticatedWithReauth;
}
(_, TransitionAction::Logout, TransitionResult::Ok) => {
self.state = State::Unauthenticated;
}
(_, _, TransitionResult::Ok) => {
unreachable!();
}
(_, _, TransitionResult::Error) => {
self.state = State::Unauthenticated;
}
}
}
}

View file

@ -0,0 +1,2 @@
pub(crate) mod auth_only;
pub(crate) mod basic;

View file

@ -50,7 +50,7 @@ enum OrcaOpt {
/// This allows deterministic regeneration of a test state file.
seed: Option<i64>,
// Todo - support the extra uris field for replicated tests.
// TODO - support the extra uris field for replicated tests.
#[clap(long = "profile")]
/// The configuration file path to update (or create)
profile_path: PathBuf,

View file

@ -30,11 +30,40 @@ async fn preflight_person(
match &person.credential {
Credential::Password { plain } => {
client
.person_set_pirmary_password_only(&person.username, plain)
.person_set_primary_password_only(&person.username, plain)
.await?;
}
}
// For each role we are part of, did we have other permissions required to fufil that?
for role in &person.roles {
if let Some(need_groups) = role.requires_membership_to() {
for group_name in need_groups {
client
.group_add_members(&group_name, &[person.username.as_str()])
.await?;
}
}
}
Ok(())
}
async fn preflight_group(client: Arc<kani::KanidmOrcaClient>, group: Group) -> Result<(), Error> {
if client.group_exists(group.name.as_str()).await? {
// Do nothing? Do we need to reset them later?
} else {
client.group_create(group.name.as_str()).await?;
}
// We can submit all the members in one go.
let members = group.members.iter().map(|s| s.as_str()).collect::<Vec<_>>();
client
.group_set_members(group.name.as_str(), members.as_slice())
.await?;
Ok(())
}
@ -45,8 +74,9 @@ pub async fn preflight(state: State) -> Result<(), Error> {
// Apply any flags if they exist.
apply_flags(client.clone(), state.preflight_flags.as_slice()).await?;
// Create persons.
let mut tasks = Vec::with_capacity(state.persons.len());
// Create persons.
for person in state.persons.into_iter() {
let c = client.clone();
tasks.push(tokio::spawn(preflight_person(c, person)))
@ -62,6 +92,19 @@ pub async fn preflight(state: State) -> Result<(), Error> {
}
// Create groups.
let mut tasks = Vec::with_capacity(state.groups.len());
for group in state.groups.into_iter() {
let c = client.clone();
tasks.push(tokio::spawn(preflight_group(c, group)))
}
for task in tasks {
task.await.map_err(|tokio_err| {
error!(?tokio_err, "Failed to join task");
Error::Tokio
})??;
}
// Create integrations.

View file

@ -22,7 +22,7 @@ async fn actor_person(
stats_queue: Arc<SegQueue<EventRecord>>,
mut actor_rx: broadcast::Receiver<Signal>,
) -> Result<(), Error> {
let mut model = person.model.as_dyn_object();
let mut model = person.model.as_dyn_object()?;
while let Err(broadcast::error::TryRecvError::Empty) = actor_rx.try_recv() {
let event = model.transition(&client, &person).await?;
@ -41,9 +41,10 @@ pub struct EventRecord {
}
pub enum EventDetail {
Authentication,
Login,
Logout,
PersonSetSelfMail,
PersonReauth,
Error,
}
@ -64,12 +65,13 @@ async fn execute_inner(
// continue.
}
_ = control_rx.recv() => {
// Untill we add other signal types, any event is
// Until we add other signal types, any event is
// either Ok(Signal::Stop) or Err(_), both of which indicate
// we need to stop immediately.
return Err(Error::Interupt);
return Err(Error::Interrupt);
}
}
info!("warmup time passed, statistics will now be collected ...");
let start = Instant::now();
if let Err(crossbeam_err) = stat_ctrl.push(TestPhase::Start(start)) {
@ -92,10 +94,11 @@ async fn execute_inner(
// continue.
}
_ = recv => {
// Untill we add other signal types, any event is
// Until we add other signal types, any event is
// either Ok(Signal::Stop) or Err(_), both of which indicate
// we need to stop immediately.
return Err(Error::Interupt);
debug!("Interrupt");
return Err(Error::Interrupt);
}
}
} else {
@ -106,7 +109,7 @@ async fn execute_inner(
if let Err(crossbeam_err) = stat_ctrl.push(TestPhase::End(end)) {
error!(
?crossbeam_err,
"Unable to signal statistics collector to start"
"Unable to signal statistics collector to end"
);
return Err(Error::Crossbeam);
}
@ -175,14 +178,14 @@ pub async fn execute(state: State, control_rx: broadcast::Receiver<Signal>) -> R
}
let warmup = state.profile.warmup_time();
let testtime = state.profile.test_time();
let test_time = state.profile.test_time();
// We run a seperate test inner so we don't have to worry about
// We run a separate test inner so we don't have to worry about
// task spawn/join within our logic.
let c_stats_ctrl = stats_ctrl.clone();
// Don't ? this, we want to stash the result so we cleanly stop all the workers
// before returning the inner test result.
let test_result = execute_inner(warmup, testtime, control_rx, c_stats_ctrl).await;
let test_result = execute_inner(warmup, test_time, control_rx, c_stats_ctrl).await;
info!("stopping stats");

View file

@ -1,10 +1,10 @@
use crate::error::Error;
use crate::model::ActorModel;
use crate::model::{ActorModel, ActorRole};
use crate::models;
use crate::profile::Profile;
use serde::{Deserialize, Serialize};
use std::collections::BTreeSet;
use std::path::Path;
/// A serialisable state representing the content of a kanidm database and potential
/// test content that can be created and modified.
///
@ -16,7 +16,7 @@ pub struct State {
// ----------------------------
pub preflight_flags: Vec<Flag>,
pub persons: Vec<Person>,
// groups: Vec<Group>,
pub groups: Vec<Group>,
// oauth_clients: Vec<Oauth2Clients>,
}
@ -55,23 +55,37 @@ pub enum Flag {
DisableAllPersonsMFAPolicy,
}
#[derive(Debug, Serialize, Deserialize)]
#[derive(Default, Debug, Serialize, Deserialize)]
pub enum PreflightState {
#[default]
Present,
Absent,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
/// A model defines *how* an actors makes it's choices. For example the choices
/// could be purely random, they could be a linear pattern, or they could have
/// some set of weights related to choices they make.
///
/// Some models can *restrict* the set of choices that an actor may make.
///
/// This compliments ActorRoles, which define the extended actions an Actor may
/// choose to perform. If ActorRoles are present, the model MAY choose to use
/// these roles to perform extended operations.
#[derive(Debug, Serialize, Deserialize, Clone, Default)]
pub enum Model {
/// This is a "hardcoded" model that just authenticates and searches
AuthOnly,
/// A simple linear executor that does actions in a loop.
#[default]
Basic,
}
impl Model {
pub fn as_dyn_object(&self) -> Box<dyn ActorModel + Send> {
match self {
Model::Basic => Box::new(crate::model_basic::ActorBasic::new()),
}
pub fn as_dyn_object(&self) -> Result<Box<dyn ActorModel + Send>, Error> {
Ok(match self {
Model::AuthOnly => Box::new(models::auth_only::ActorAuthOnly::new()),
Model::Basic => Box::new(models::basic::ActorBasic::new()),
})
}
}
@ -85,7 +99,15 @@ pub struct Person {
pub preflight_state: PreflightState,
pub username: String,
pub display_name: String,
pub member_of: BTreeSet<String>,
pub roles: BTreeSet<ActorRole>,
pub credential: Credential,
pub model: Model,
}
#[derive(Default, Debug, Serialize, Deserialize)]
pub struct Group {
pub name: String,
pub preflight_state: PreflightState,
pub role: ActorRole,
pub members: BTreeSet<String>,
}

View file

@ -47,6 +47,7 @@ impl DataCollector for BasicStatistics {
break start;
}
Some(TestPhase::End(_)) => {
error!("invalid state");
// Invalid state.
return Err(Error::InvalidState);
}
@ -69,6 +70,7 @@ impl DataCollector for BasicStatistics {
break end;
}
Some(TestPhase::StopNow) => {
warn!("requested to stop now!");
// We have been told to stop immediately.
return Ok(());
}
@ -76,6 +78,8 @@ impl DataCollector for BasicStatistics {
}
};
info!("start statistics processing ...");
let mut count: usize = 0;
let mut optimes = Vec::new();