From f0dfe4c70d177a6ff266ba35ff2994d5dd41dadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Thu, 11 Jan 2024 11:21:56 +0100 Subject: [PATCH 01/32] Add probula to the mix --- build.sbt | 5 +- src/main/scala/probula/Dist.scala | 729 ++++++++++++++++++++++ src/main/scala/probula/IData.scala | 105 ++++ src/main/scala/probula/Name.scala | 55 ++ src/main/scala/probula/Probula.scala | 112 ++++ src/test/scala/probula/Probula.test.scala | 135 ++++ 6 files changed, 1139 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/probula/Dist.scala create mode 100644 src/main/scala/probula/IData.scala create mode 100644 src/main/scala/probula/Name.scala create mode 100644 src/main/scala/probula/Probula.scala create mode 100644 src/test/scala/probula/Probula.test.scala diff --git a/build.sbt b/build.sbt index 004808fc..a1c4d338 100644 --- a/build.sbt +++ b/build.sbt @@ -9,8 +9,8 @@ val catsVersion = "2.6.1" scalacOptions ++= Seq ( "-deprecation", "-feature", - "-Yindent-colons", - // "-source:future", Disabled for stryker + "-source:future", // doesn't work with stryker + "-language:adhocExtensions", ) @@ -25,6 +25,7 @@ libraryDependencies ++= Seq ( "org.typelevel" %% "discipline-scalatest" % "2.1.5", "org.typelevel" %% "paiges-core" % "0.4.2", "org.scalanlp" %% "breeze" % "2.1.0", + "org.typelevel" %% "spire" % "0.18.0" // for probula ) Test / parallelExecution := false diff --git a/src/main/scala/probula/Dist.scala b/src/main/scala/probula/Dist.scala new file mode 100644 index 00000000..f671a150 --- /dev/null +++ b/src/main/scala/probula/Dist.scala @@ -0,0 +1,729 @@ +/** A tiny pure Bayesian inference framework based on rejection sampling. + * A small alternative for the core part of Figaro. + * + * Copyright (c) 2023 Andrzej Wasowski + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +/** A tiny pure Bayesian inference framework based on rejection + * sampling. A small alternative for the core part of Figaro. + */ +package probula + +import scala.annotation.targetName + +import spire.random.Uniform.* +import spire.random.Dist.* +import spire.random.* + +type RNG = spire.random.rng.SecureJava + +/** A representation of probabilistic models as multivariate + * distributions, effectively hierarchical Bayesian models. + */ +trait Dist[+T]: + self => + + // Abstract members + + /** The name of the random variable, represented using a qualified + * name term. For a multivariate model, the name is typically + * interpreted as the name of the last variable added (the last in + * the tuple). This is not ideal and will have to be redesigned in + * the future. + * + * @see [[probula.Name]] + */ + def name: Name + + /** Generate a sample from `this` model. The sample is an infinite + * lazy list (underneath). Typically this method is not used by + * users of the library. They will use the variant that takes a + * sample size. + */ + protected def sample[U >: T](using RNG): IData[U] + + // Derived members + + /** Generate a sample from `this` model containing `n` samples. + */ + def sample[U >: T](n: SampleSize)(using RNG): IData[U] = + val idata = this.sample(using summon[RNG]) + idata.copy(chain = idata.chain.take(n)) + + /** Create a univariate[1] model by mapping a function on the `this` + * model. Users should typically use the `detDep` function + * instead, as `map` "hides" all the existing variables in the + * model, and the resulting model is univariate, regardless how + * many variables `this` had. + * + * [1] Technically the resulting model does not have to be + * univariate, even though it typically is. It will have the arity + * of type `U`. + * + * @param name The descriptive name of the only variable in the new + * model. + * @see [[detDep]] + */ + def map[U](l: Name)(f: T => U): Dist[U] = new Dist[U]: + val name = l + def sample[V >: U](using rng: RNG): IData[V] = + self.sample.map(f).label(l) + + /** Create a univariate[1] model by mapping a function on the `this` + * model. Users should typically use the `detDep` function + * instead, as `map` "hides" all the existing variables in the + * model, and the resulting model is univariate, regardless how + * many variables `this` had. + * + * [1] Technically the resulting model does not have to be + * univariate, even though it typically is. It will have the arity + * of type `U`. + * + * @param name The descriptive name of the only variable in the new + * model. + * @see [[detDep]] + */ + def map[U](l: String)(f: T => U): Dist[U] = + self.map(l.toName)(f) + + /** Create a univariate[1] model by mapping a function on the `this` + * model. Users should typically use the `detDep` function + * instead, as `map` "hides" all the existing variables in the + * model, and the resulting model is univariate, regardless how + * many variables `this` had. + * + * The variable in the model resulting model will have the same + * descriptive name as the `this` node had, but suffixed with + * `map`. + * + * [1] Technically the resulting model does not have to be + * univariate, even though it typically is. It will have the arity + * of type `U`. + * + * @see [[detDep]] + */ + def map[U](f: T => U): Dist[U] = + val l = Name.Suffixed(self.name, "map") + self.map(l)(f) + + /** Add a new variable (dimension) to `this` model by mapping + * function `f`. The new variable is just added to the current + * model, so the arity of the model increases by one. Users should + * use [[detDep]] instead. + * + * @param name The descriptive name of the newly added variable + * @see [[detDep]] + */ + def _map[U](l: Name)(f: T => U): Dist[(T,U)] = + self.map { t => (t, f(t)) } + .label(self.name -> l) + + /** Add a new variable (dimension) to `this` model by mapping + * function `f`. The new variable is just added to the current + * model, so the arity of the model increases by one. Users should + * use [[detDep]] instead. + * + * @param name The descriptive name of the newly added variable + * @see [[detDep]] + */ + def _map[U](l: String)(f: T => U): Dist[(T,U)] = + self._map(l.toName)(f) + + /** Add a new variable (dimension) to `this` model by mapping + * function `f`. The new variable is just added to the current + * model, so the arity of the model increases by one. Users should + * use [[detDep]] instead. + * + * The new variable will be using the same name as `this` but + * suffixed with `_map`. + * + * @see [[detDep]] + */ + def _map[U](f: T => U): Dist[(T,U)] = + val newName = this.name -> Name.Suffixed(name, "_map") + self._map(newName)(f) + + + def map2[U, V](other: Dist[U]) (f: (T,U) => V): Dist[V] = new Dist[V]: + val name: Name = Name.Suffixed (self.name -> other.name, "map2") + def sample[W >: V](using rng: RNG): IData[W] = + self.sample.map2(other.sample)(f) + .label(this.name) + + def map2[U, V](other: Dist[U]) (f: ((T,U)) => V): Dist[V] = + self.map2(other)(scala.Function.untupled(f)) + + def _map2[U, V](other: Dist[U]) (f: (T,U) => V): Dist[(T,U,V)] = + self.map2(other)((t,u) => (t, u, f(t,u))) + def _map2[U, V](that: Dist[U]) (f: ((T,U)) => V): Dist[(T,U,V)] = + _map2(that)(scala.Function.untupled(f)) + + /** Zip two variables into one bivariate variables. This should + * typically be used only for variables constructed with basic + * constructors (independent). + */ + def zip[U](that: Dist[U]): Dist[(T,U)] = + this.map2(that)(identity[(T,U)]) + + /** Same as `zip`. */ + infix def -> [U](that: Dist[U]): Dist[(T,U)] = + this.zip(that) + + /** Create a univariate[1] model by mapping function `f` the `this` + * model. + * + * Users should typically use the `probDep` function + * instead, as `flatMap` "hides" all the existing variables in the + * model, and the resulting model is univariate, regardless how + * many variables `this` had. + * + * [1] Technically the resulting model does not have to be + * univariate, even though it typically is. It will have the arity + * of type `U`. + * + * @param l The descriptive name of the only variable in the new + * model. + * @param f The function to be mapped. + * @return The model created by mapping. All original variables + * are hidden (lost) + * @see [[probDep]] + */ + def flatMap[U](l: Name)(f: T => Dist[U]): Dist[U] = new Dist: + def name: Name = l + def sample[S >: U](using RNG): IData[S] = + self.sample.flatMap(t => f(t).sample(1)) + + /** Create a univariate[1] model by mapping function `f` the `this` + * model. + * + * Users should typically use the `probDep` function + * instead, as `flatMap` "hides" all the existing variables in the + * model, and the resulting model is univariate, regardless how + * many variables `this` had. + * + * [1] Technically the resulting model does not have to be + * univariate, even though it typically is. It will have the arity + * of type `U`. + * + * @param l The descriptive name of the only variable in the new + * model. + * @param f The function to be mapped. + * @return The model created by mapping. All original variables + * are hidden (lost) + * @see [[probDep]] + */ + def flatMap[U](l: String)(f: T => Dist[U]): Dist[U] = + self.flatMap(l.toName)(f) + + + /** Create a univariate[1] model by mapping function `f` on `this` + * model. + * + * Users should typically use the `probDep` function + * instead, as `flatMap` "hides" all the existing variables in the + * model, and the resulting model is univariate, regardless how + * many variables `this` had. + * + * [1] Technically the resulting model does not have to be + * univariate, even though it typically is. It will have the arity + * of type `U`. + * + * @param f The function to be mapped. + * @return The model created by mapping. All original variables + * are hidden (lost). The new model carries the same + * name as `this` but suffixed with `flatMap`. + * @see [[probDep]] + */ + def flatMap[U](f: T => Dist[U]): Dist[U] = + self.flatMap(Name.Suffixed(self.name, "flatMap"))(f) + + /** Add a new variable to the `this` model by flatMapping function + * `f`. + * + * Users should typically use the `probDep` function + * instead, as `_flatMap`. The resulting model has arity higher by + * 1 than `this`. + * + * @param l The descriptive name of the only variable in the new + * model. + * @param f The function to be flatMapped to derive the new + * variable. + * @return The model created by flatMapping and adding the + * result to the existing model tuple. + * @see [[probDep]] + */ + def _flatMap[U](l: Name)(f: T => Dist[U]): Dist[(T,U)] = + self.flatMap { t => f(t).map { u => (t, u) } } + + /** Add a new variable to the `this` model by flatMapping function + * `f`. + * + * Users should typically use the `probDep` function + * instead, as `_flatMap`. The resulting model has arity higher by + * 1 than `this`. + * + * @param l The descriptive name of the only variable in the new + * model. + * @param f The function to be flatMapped to derive the new + * variable. + * @return The model created by flatMapping and adding the + * result to the existing model tuple. + * @see [[probDep]] + */ + def _flatMap[U](l: String)(f: T => Dist[U]): Dist[(T,U)] = + self._flatMap(l.toString)(f) + + /** Add a new variable to the `this` model by flatMapping function + * `f`. + * + * Users should typically use the `probDep` function + * instead, as `_flatMap`. The resulting model has arity higher by + * 1 than `this`. The new variable is given no name. + * + * @param f The function to be flatMapped to derive the new + * variable. + * @return The model created by flatMapping and adding the + * result to the existing model tuple. + * @see [[probDep]] + */ + def _flatMap[U](f: T => Dist[U]): Dist[(T,U)] = + self._flatMap(Name.No)(f) + + /** Change the name of the current model/variable to an arbitrary + * string `l`. + */ + def label(l: String): Dist[T] = + self.label(l.toName) + + /** Change the name of the current model/variable to an arbitrary + * new name `l`. + */ + def label(l: Name): Dist[T] = new Dist[T]: + override def name: Name = l + override def sample[S >: T](using rng: RNG): IData[S] = + self.sample.label(name) + + /** Reject all the values of the model that do not satisfy `p`. + * + * This is also called observe in literature (and since we only do + * rejection sampling for now it is equivalent to conditioning). + * The new filtered model will have the same name as `this` but + * suffixed with `filter`. + */ + def filter(p: T => Boolean): Dist[T] = new Dist[T]: + def name = Name.Suffixed(self.name, "filter") + override def sample[S >: T](using rng: RNG): IData[S] = + self.sample.filter(p) + + /** Reject all the values of the model that differ from `value` + * usign a usual equality test `==`. + * + * This is also called observe in literature (and since we only do + * rejection sampling for now it is equivalent to conditioning). + * The new filtered model will have the same name as `this` but + * suffixed with `filter`. + */ + def filter[S >: T](value: S): Dist[T] = + self.filter { _ == value } + + // Disabled, as the semantics is unexpected + // def withFilter(p: T => Boolean): WithFilter = new WithFilter(p) + + /** Not used presently. */ + class WithFilter(p: T => Boolean): + def map[U](f: T => U): Dist[U] = + self.filter(p).map(f) + def withFilter(q: T => Boolean): WithFilter = + WithFilter(x => p(x) && q(x)) + + /** Reject all the values of the model that do not satisfy `p`. + * + * This is also called observe in literature (and since we only do + * rejection sampling for now it is equivalent to conditioning). + * The new filtered model will have the same name as `this` but + * suffixed with `filter`. + * + * The same as [[filter]]. + */ + def condition(p: T => Boolean): Dist[T] = + self.filter(p) + + /** Reject all the values of the model that differ from `value` + * usign a usual equality test `==`. + * + * This is also called observe in literature (and since we only do + * rejection sampling for now it is equivalent to conditioning). + * The new filtered model will have the same name as `this` but + * suffixed with `filter`. + * + * The same as [[filter]]. + */ + def condition[S >: T](value: S): Dist[T] = + self.filter(value) + + /** Reject all the values of the model that do not match the pattern + * in the partial fucntion `f` (so more precisely for which `f` is + * not defined). In practice `f` is a _case+ claose without a body + * as the body is not executed by `matching`. + * + * This is also called observe in literature (and since we only do + * rejection sampling for now it is equivalent to conditioning). + * The new filtered model will have the same name as `this` but + * suffixed with `filter`. + */ + def matching[A](f: PartialFunction[T, A]): Dist[T] = + self.filter(f.isDefinedAt) + + +// NB. Overloaded extension methods cannot compete with +// trait methods (apparentely then the trait methods are +// tried only) - for these reason even Dist[T] must use extensions, +// so that we can support other arities. +extension [T](self: Dist[T]) + + /** Define a new variable with name `l` that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` with name. + */ + def detDep[U](l: Name)(f: T => U): Dist[(T, U)] = + self._map(l)(f) + + /** Define a new variable (without a name) that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` + */ + def detDep[U](f: T => U): Dist[(T, U)] = + self._map(f) + + /** Define a new variable with name `l` that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` with name. + */ + def detDep[U](l: String)(f: T => U): Dist[(T,U)] = + self._map(l)(f) + + /** Define a new variable with name `l` that probabilistically + * depends on the values of existing variables in the model. + * Does the same thing as `_flatMap` with name. + */ + def probDep[U](l: Name)(f: T => Dist[U]): Dist[(T, U)] = + self._flatMap(l)(f) + + /** Define a new variable with name `l` that probabilistically + * depends on the values of existing variables in the model. + * Does the same thing as `_flatMap` with name. + */ + def probDep[U](l: String)(f: T => Dist[U]): Dist[(T, U)] = + self.probDep(l.toName)(f) + + /** Define a new variable with a generated name that probabilistically + * depends on the values of existing variables in the model. + * Does the same thing as `_flatMap` (The generated name is different). + */ + def probDep[U](f: T => Dist[U]): Dist[(T, U)] = + self.probDep(Name.Suffixed(self.name, "probDep"))(f) + + /** Add a new uniformly distributed variable to a model. */ + def uniform[U](name: Name)(values: U*): Dist[(T,U)] = for + t <- self + u <- Uniform[U](name)(values*) + yield (t, u) + + /** Add a new uniformly distributed variable to a model. */ + def uniform[U](name: String)(values: U*): Dist[(T,U)] = + self.uniform(name.toName)(values*) + + /** Add a new uniformly distributed variable to a model. */ + def uniform[U](values: U*): Dist[(T,U)] = + self.uniform(Name.No)(values*) + + /** Add a new Bernoulli variable to a model. */ + def bernoulli[U](name: Name)(p: Double, success: U, failure: U): Dist[(T, U)] = for + t <- self + u <- Bernoulli(name, p, success, failure) + yield (t, u) + + /** Add a new Bernoulli variable to a model. */ + def bernoulli[U](name: String)(p: Double, success: U, failure: U): Dist[(T, U)] = + self.bernoulli(name.toName)(p, success, failure) + + /** Add a new Bernoulli variable to a model. */ + def bernoulli[U](p: Double, success: U, failure: U): Dist[(T, U)] = + self.bernoulli(Name.No)(p, success, failure) + + /** Add a new Bernoulli variable to a model. */ + def bernoulli[U](name: Name)(f: T => Double, success: U, failure: U) + : Dist[(T, U)] = for + t <- self + p = f(t) + u <- Bernoulli(name, p, success, failure) + yield (t, u) + + /** Add a new Bernoulli variable to a model. */ + def bernoulli[U](name: String)(f: T => Double, success: U, failure: U) + : Dist[(T, U)] = + self.bernoulli(name.toName)(f, success, failure) + + /** Add a new Bernoulli variable to a model. */ + def bernoulli[U](f: T => Double, success: U, failure: U) + : Dist[(T, U)] = + self.bernoulli(Name.No)(f, success, failure) + + +// This "tupled causes some problems so I would like to remove it" +extension [S, T](self: Dist[(S, T)]) + + /** Define a new variable with name `l` that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` with name. + */ + @targetName("detDep2name") + def detDep[U](l: Name)(f: (S, T) => U) + : Dist[(S, T, U)] = + self._map(l)(f.tupled).declutter + + /** Define a new variable with name `l` that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` with name. + */ + @targetName("detDep2string") + def detDep[U](l: String)(f: (S, T) => U): Dist[(S, T, U)] = + self.detDep[U](l.toName)(f) + + /** Define a new variable without a name that deterministically + * depends on the values of existing variables in the model. + * Behaves consistently with `_map` without a name - it reuses the + * name from `this` and suffixes it with `detDep`. + */ + @targetName("detDep2") + def detDep[U](f: (S, T) => U): Dist[(S, T, U)] = + self.detDep[U](Name.Suffixed(self.name, "detDep"))(f) + + /** Define a new variable with name `l` that probabilistically + * depends on the values of existing variables in the model. + * Does the same thing as `_flatMap` with name. + */ + @targetName("probDep2name") + def probDep[U](l: Name)(f: (S, T) => Dist[U]): Dist[(S, T, U)] = + self._flatMap(l)(f.tupled).declutter + + /** Define a new variable with name `l` that probabilistically + * depends on the values of existing variables in the model. + * Does the same thing as `_flatMap` with name. + */ + @targetName("probDep2string") + def probDep[U](l: String)(f: (S, T) => Dist[U]): Dist[(S, T, U)] = + self.probDep[U](l.toName)(f) + + /** Define a new variable with a generated name that probabilistically + * depends on the values of existing variables in the model. + * Does the same thing as `_flatMap` (The generated name is different). + */ + @targetName("probDep2") + def probDep[U](f: (S, T) => Dist[U]): Dist[(S, T, U)] = + self.probDep[U](Name.Suffixed(self.name, "probDep"))(f) + + /** Simplify the model to only show the first variable. */ + @targetName("firstOfTwo") + def _1: Dist[S] = self.map { _._1 } + + /** Simplify the model to only show the second variable. */ + @targetName("secondOfTwo") + def _2: Dist[T] = self.map { _._2 } + + /** Add a new uniform variable to a model. */ + @targetName("uniform2name") + def uniform[U](name: Name)(values: U*): Dist[(S, T, U)] = for + (s,t) <- self + u <- Uniform[U](name)(values*) + yield (s, t, u) + + /** Add a new uniform variable to a model. */ + @targetName("uniform2string") + def uniform[U](name: String)(values: U*): Dist[(S, T, U)] = + self.uniform(name.toName)(values*) + + /** Add a new Bernoulli variable to a model. */ + @targetName("bernoulli2name") + def bernoulli[U](name: Name)(p: Double, success: U, failure: U): Dist[(S, T, U)] = for + (s, t) <- self + u <- Bernoulli(name, p, success, failure) + yield (s, t, u) + + /** Add a new Bernoulli variable to a model. */ + @targetName("bernoulli2string") + def bernoulli[U](name: String)(p: Double, success: U, failure: U): Dist[(S, T, U)] = + self.bernoulli(name.toName)(p, success, failure) + + /** Add a new Bernoulli variable to a model. */ + @targetName("bernoulli2") + def bernoulli[U](p: Double, success: U, failure: U): Dist[(S, T, U)] = + self.bernoulli(Name.No)(p, success, failure) + + /** Add a new Bernoulli variable to a model. */ + @targetName("bernoulliDep2name") + def bernoulli[U](name: Name)(success: U, failure: U)(f: (S, T) => Double) + : Dist[(S, T, U)] = for + (s, t) <- self + p = f(s, t) + u <- Bernoulli(name, p, success, failure) + yield (s, t, u) + + /** Add a new Bernoulli variable to a model. */ + @targetName("bernoulliDep2string") + def bernoulli[U](name: String)(success: U, failure: U)(f: (S, T) => Double) + : Dist[(S, T, U)] = + self.bernoulli(name.toName)(success, failure)(f) + + /** Add a new Bernoulli variable to a model. */ + @targetName("bernoulliDep2") + def bernoulli[U](success: U, failure: U)(f: (S, T) => Double) + : Dist[(S, T, U)] = + self.bernoulli(Name.No)(success, failure)(f) + +extension [T1, T2, T3](self: Dist[(T1, T2, T3)]) + + /** Define a new variable with name `l` that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` with name. + */ + @targetName("detDep3name") + def detDep[T](l: Name)(f: (T1, T2, T3) => T) + : Dist[(T1, T2, T3, T)] = + self._map(l)(f.tupled).declutter + + /** Define a new variable with name `l` that deterministically + * depends on the values of existing variables in the model. + * Does the same thing as `_map` with name. + */ + @targetName("detDep3string") + def detDep[T](l: String)(f: (T1, T2, T3) => T) + : Dist[(T1, T2, T3, T)] = + self.detDep(l.toName)(f) + + /** Define a new variable without a name that deterministically + * depends on the values of existing variables in the model. + * Behaves consistently with `_map` without a name - it reuses the + * name from `this` and suffixes it with `detDep`. + */ + @targetName("detDep3tupled") + def detDep[T](f: (T1, T2, T3) => T) + : Dist[(T1, T2, T3, T)] = + self.detDep(Name.Suffixed(self.name, "map"))(f) + + /** Simplify the model to only show the first variable. */ + @targetName("firstOfThree") + def _1: Dist[T1] = self.map { _._1 } + + /** Simplify the model to only show the second variable. */ + @targetName("secondOfThree") + def _2: Dist[T2] = self.map { _._2 } + + /** Simplify the model to only show the third variable. */ + @targetName("thirdOfThree") + def _3: Dist[T3] = self.map { _._3 } + + + +extension [T1, T2, T3, T4](ego: Dist[(T1, T2, T3, T4)]) + + /** Simplify the model to only show the third variable. */ + @targetName("thirdOfFour") + def _3: Dist[T3] = ego.map { _._3 } + + /** Simplify the model to only show the fourth variable. */ + @targetName("fourthOfFour") + def _4: Dist[T4] = ego.map { _._4 } + + + +extension [S, T, U](ego: Dist[((S,T),U)]) + /** Flatten the model tuple to a triple. */ + @targetName("declutter2") + def declutter: Dist[(S, T, U)] = + ego.map { case ((s,t),u) => (s, t, u) } + +extension [T1, T2, T3, U](ego: Dist[((T1, T2, T3), U)]) + /** Flatten the model tuple to a quadruple. */ + @targetName("declutter3") + def declutter: Dist[(T1, T2, T3, U)] = + ego.map { case ((t1, t2, t3), u) => (t1, t2, t3, u) } + +// Fixed concrete distributions (roots) + +case class Dirac[T](name: Name, value: T) + extends Dist[T]: + def sample[S >: T](using RNG): IData[S] = + IData(this.name, LazyList.continually(value)) + +object Dirac: + def apply[T](value: T): Dirac[T] = + new Dirac[T](Name.No, value) + +case class Bernoulli[T](name: Name, p: Double, success: T, failure: T) + extends Dist[T]: + + private lazy val gen = + spire.random.Uniform[Double](0.0, 1.0) + .map[T] { x => if x <= p then success else failure } + + def sample[S >: T](using rng: RNG): IData[S] = + val chain = gen.toLazyList(rng) + IData(this.name, chain) + +object Bernoulli: + def apply(p: Double = 0.5): Bernoulli[Boolean] = + new Bernoulli[Boolean](Name.No, p, true, false) + def apply[T](p: Double, success: T, failure: T): Bernoulli[T] = + new Bernoulli[T](Name.No, p, success, failure) + def apply[T](l: String, p: Double, success: T, failure: T): Bernoulli[T] = + new Bernoulli[T](l.toName, p, success, failure) + + def apply(l: String, p: Double): Bernoulli[Boolean] = + new Bernoulli[Boolean](l.toName, p, true, false) + def apply(l: Name, p: Double): Bernoulli[Boolean] = + new Bernoulli[Boolean](l, p, true, false) + + +case class Uniform[+T](name: Name) (values: T*) + extends Dist[T]: + + val rangeMin: Int = 0 + val rangeMax: Int = values.length - 1 + + private lazy val indices = rangeMin to rangeMax + lazy val valueMap: Map[Int, T] = (indices zip values).toMap + + private lazy val gen = spire.random.Uniform(rangeMin, rangeMax) + + def sample[S >: T](using rng: RNG): IData[S] = + val chain = gen.toLazyList(rng) + .map(valueMap) + IData(this.name, chain) + +object Uniform: + // There is certainly a much more efficient way to implement this + // for large ranges + def apply(name: String)(min: Int, max: Int): Uniform[Int] = + new Uniform[Int](name.toName)(min to max *) + + def apply(min: Int, max: Int): Uniform[Int] = + new Uniform[Int](Name.No)(min to max*) + + def apply[T](name: String)(values: T*): Uniform[T] = + new Uniform[T](name.toName)(values*) + + def apply[T](values: T*): Uniform[T] = + new Uniform[T](Name.No)(values*) diff --git a/src/main/scala/probula/IData.scala b/src/main/scala/probula/IData.scala new file mode 100644 index 00000000..5bddd73e --- /dev/null +++ b/src/main/scala/probula/IData.scala @@ -0,0 +1,105 @@ +/* A tiny pure Bayesian inference framework based on rejection sampling. + * A small alternative for the core part of Figaro. + * + * Copyright (c) 2023 Andrzej Wasowski + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +/** A tiny pure Bayesian inference framework based on rejection sampling. + * A small alternative for the core part of Figaro. + */ +package probula + +type SampleSize = Int +type Chain[T] = LazyList[T] + +/** Representation of Inference Data. The intention is to make it + * compatible with the inference data in the Python world (oen day...) + */ +case class IData[+T](name: Name, chain: Chain[T]): + + /** The sameple size */ + lazy val size: Int = this.chain.size + + private[probula] def map[S](f: T => S): IData[S] = + IData(this.name, this.chain.map(f)) + + private[probula] def flatMap[S](f: T => IData[S]): IData[S] = + val newChain: Chain[S] = + this.chain.map[S] { t => f(t).chain.head } + this.copy (chain = newChain) + + private[probula] def map2[S,U](that: IData[S])(f: (T,S) => U): IData[U] = + val newChain = this.chain.zip(that.chain).map(f.tupled) + IData(Name.No, newChain) + + private[probula] def filter(p: T => Boolean): IData[T] = + this.copy(chain = this.chain.filter(p)) + + def label(name: Name): IData[T] = + this.copy (name = name) + + def label(name: String): IData[T] = + label(name.toName) + + override def toString : String = + s"""${this.name.toString}: ${this.chain.mkString(",")}""" + + /** Estimate the probability of an event defined by the predicate `p`. + * Precondition - the sample size is finite. + **/ + def probability[S >: T](p: S => Boolean): Double = + chain.filter(p).length.toDouble / this.size.toDouble + + /** Estimate the probability of an event defined by `value`. + * Precondition - the sample size is finite. + **/ + def probability[S >: T](value: S): Double = + this.probability { _ == value } + + /** Same as `probability`. */ + def pr[S >: T](p: S => Boolean): Double = + this.probability(p) + + /** Same as `probability`. */ + def pr[S >: T](value: S): Double = + this.probability(value) + + /** What is the probability of the sample part that matches the + * given case pattern? + */ + def prMatching[A] (f: PartialFunction[T, A]): Double = + this.pr(f.isDefinedAt) + +import math.Numeric.Implicits.infixNumericOps +extension [T](ego: IData[T]) + + /** Compute a median of a sampole with a defined `Ordering`. + * + * For simplicity we drop one element if the sample is of even + * length. Typically to be used on a univariate sample of numbers + * (then the ordering exists). + */ + def median(using Ordering[T]): T = + val N = ego.size + val M = N - (N % 2) + val s = (if (N != M) ego.chain.tail else ego.chain).sorted + s(M / 2) + + /** Compute a mean for a numeric sample. */ + def mean(using math.Numeric[T]): Double = + ego.chain.sum.toDouble / ego.size + + /** Same as `mean` */ + def expectedValue(using math.Numeric[T]): Double = mean diff --git a/src/main/scala/probula/Name.scala b/src/main/scala/probula/Name.scala new file mode 100644 index 00000000..1b5db8ea --- /dev/null +++ b/src/main/scala/probula/Name.scala @@ -0,0 +1,55 @@ +/** A tiny pure Bayesian inference framework based on rejection sampling. + * A small alternative for the core part of Figaro. + * + * Copyright (c) 2023 Andrzej Wasowski + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package probula + +enum Name: + case Simple(name: String) + case Tuple(names: Name*) + case Suffixed(stem: Name, suffix: String) + case No + + override def toString: String = this match + case Simple(name) => + name + + case Tuple(names*) => + names + .map { _.toString } + .mkString("[", ",", "]") + + case Suffixed(stem, suffix) => + s"$stem.$suffix" + + case No => "■" + + infix def -> (that: Name) = + Name.Tuple (this, that) + + infix def :: (other: Name) = other match + case Name.Tuple(names*) => + Name.Tuple((names.prepended (this))*) + case _ => this -> other + +extension (s: String) + def toName: Name = Name.Simple(s) +extension (names: (Name, Name)) + def toName: Name = Name.Tuple(names.toList*) +extension (names: (Name, Name, Name)) + def toName: Name = Name.Tuple(names.toList*) + + diff --git a/src/main/scala/probula/Probula.scala b/src/main/scala/probula/Probula.scala new file mode 100644 index 00000000..f50ff1e3 --- /dev/null +++ b/src/main/scala/probula/Probula.scala @@ -0,0 +1,112 @@ +/* A tiny pure Bayesian inference framework based on rejection sampling. + * A small alternative for the core part of Figaro. + * + * Copyright (c) 2023 Andrzej Wasowski + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +/** A tiny pure Bayesian inference framework based on rejection + * sampling. A small alternative for the core part of Figaro. + */ +package probula + + +/** The Probula object is used as an entry point to the framework. One + * starts building a probula probabilistic model by using factory + * methods from this object. Once the model is initiated (the first + * variable is created), one can use the model methods (@see + * probula.Dist) to add more new variables, hierarchical dependencies, + * and observations. + */ +object Probula: + + /** Create a model with a single uniformly distributed discrete + * variable named `name` (for printing). + * + * @param name A symbolic name of the variable (used for + * presentation) + * @param values A variable length argument listing enumerating the + * values of the uniform distribution. + * @return A model of type `Dist[T]` that has a single random + * variable + */ + def uniform[T](name: Name)(values: T*): Uniform[T] = + Uniform[T](name)(values*) + + /** Create a model with a single uniformly distributed discrete + * random variable named `name` (for printing). + * + * @param name A symbolic name of the variable (used for + * presentation) + * @param values A variable length argument listing enumerating the + * values of the uniform distribution. + * @return A model of type `Dist[T]` that has a single random + * variable + */ + def uniform[T](name: String)(values: T*): Uniform[T] = + this.uniform(name.toName)(values*) + + /** Create a model with a single uniformly distributed discrete + * random variable without a descriptive name. + * + * @param values A variable length argument listing enumerating the + * values of the uniform distribution. + * @return A model of type `Dist[T]` that has a single random + * variable + */ + def uniform[T](values: T*): Dist[T] = + this.uniform(Name.No)(values*): Uniform[T] + + /** Create a model with a single Bernoulli-distributed discrete + * random variable named `name` (for printing). + * + * @param name A symbolic name of the variable (used for + * presentation) + * @param p The probability of success + * @param success The value used as a success outcome (arrives with + * probability `p`) + * @param faailure The value used as a failure outcome + * @return A model of type `Dist[T]` that has a single random + * variable + */ + def bernoulli[T](name: Name)(p: Double, success: T, failure: T): Bernoulli[T] = + Bernoulli[T](name, p, success, failure) + + /** Create a model with a single Bernoulli-distributed discrete + * random variable named `name` (for printing). + * + * @param name A symbolic name of the variable (used for + * presentation) + * @param p The probability of success + * @param success The value used as a success outcome (arrives with + * probability `p`) + * @param faailure The value used as a failure outcome + * @return A model of type `Dist[T]` that has a single random + * variable + */ + def bernoulli[T](name: String)(p: Double, success: T, failure: T): Bernoulli[T] = + this.bernoulli(name.toName)(p, success, failure) + + /** Create a model with a single Bernoulli-distributed discrete + * random variable without a descriptive name. + * + * @param p The probability of success + * @param success The value used as a success outcome (arrives with + * probability `p`) + * @param faailure The value used as a failure outcome + * @return A model of type `Dist[T]` that has a single random + * variable + */ + def bernoulli[T](p: Double, success: T, failure: T): Bernoulli[T] = + this.bernoulli(Name.No)(p, success, failure) diff --git a/src/test/scala/probula/Probula.test.scala b/src/test/scala/probula/Probula.test.scala new file mode 100644 index 00000000..de167aa7 --- /dev/null +++ b/src/test/scala/probula/Probula.test.scala @@ -0,0 +1,135 @@ +/** A tiny pure Bayesian inference framework based on rejection sampling. + * A small alternative for the core part of Figaro. + * + * Copyright (c) 2023 Andrzej Wasowski + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package probula +package test + +import org.scalacheck.Prop.forAll +import org.scalacheck.Prop + +given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply + + +object ProbulaSpec + extends org.scalacheck.Properties("probula"): + + /** A fake forAll, to rerun tests multiple times with various seeds. */ + def many (prop: => Prop): Prop = + forAll { (n: Int) => prop } + + val N = 10 + val M = 500 + + val x = Uniform(0, 1, 2, 3, 4, 5) + val y = Uniform(20, 21, 22) + + property("010 _mapped variables are dependent") = many: + val model: Dist[(Int, Int)] = x._map { _ + 1 } + val sample: IData[(Int, Int)] = model.sample(N) + sample.chain.forall { (x,y) => x == y - 1 } + + property("020 _map2ed variables are dependent") = many: + val model: Dist[(Int, Int, Int)] = x._map2(y) { _ + _ } + val sample: IData[(Int, Int, Int)] = model.sample(N) + sample.chain.forall { (x,y,z) => x + y == z } + + property("030 mapped variables are independent") = many: + val y = x.map { _ + 1 } + val model: Dist[(Int, Int)] = x -> y + val sample: IData[(Int, Int)] = model.sample(M) + sample.chain.exists { (x,y) => x != y - 1 } + + property("040 _map2ed variables are independent") = many: + val z: Dist[Int] = x.map2(y) { _ + _ } + val model: Dist[((Int, Int), Int)] = x -> y -> z + val sample: IData[((Int, Int), Int)] = model.sample(M) + sample.chain.exists { case ((x, y), z) => x + y != z } + + property("050 x -> x are independent (if x not-deterministic)") = many: + val model: Dist[(Int, Int)] = x -> x + val sample: IData[(Int, Int)] = model.sample(M) + sample.chain.exists { (x,y) => x != y } + + property("060 flatMapped variables are dependent") = many: + def f (x: Int): Dist[Int] = + Uniform.apply(List(-4*x, -2*x)*) + val model: Dist[Int] = x.flatMap(f) + val sample: IData[Int] = model.sample(M) + sample.chain.forall { _.abs % 2 == 0 } + && sample.chain.forall { x => x >= -20 && x <= 0} + && (0 to 5).forall { n => sample.chain.exists { _ == -4 * n } } + && (0 to 5).forall { n => sample.chain.exists { _ == -2 * n } } + + property("070 Uniform(a) is Dirac") = many: + val model = Uniform(42) + val sample = model.sample(M) + sample.chain.forall { _ == 42 } + + property("090 x -> x gives an independent pair") = + val model: Dist[(Int, Int)] = x -> x + val sample: IData[(Int, Int)] = model.sample(M) + sample.chain.exists { (x, y) => x != y } + + property("110 conditioning removes false values") = many: + val model = y.condition { _ == 20 } + val sample = model.sample(N) + sample.chain.forall { _ == 20 } + + property("120 conditioning makes condition prob. 1") = + val model = y.condition { _ == 20 } + val sample = model.sample(N) + sample.probability { _ == 20 } == 1.0 + + // Commented out tests that are not implemented to reduce confusion + // property("130 declutter also cleans up the name structure") = + // false + + // property("140 Bernoulli behaves well with p = 0 ") = false + // property("150 Bernoulli behaves well with p = 1.0 ") = false + // property("160 Bernoulli behaves well with p = 0.5 ") = false + +/* These tests are disabled, as this will not work with a clean + * for-yield. Requires more work. + + property("170 withFilter behaves like filter (1 variable)") = + val model = for x <- Bernoulli() if x yield x + model.sample(N).chain.forall(identity[Boolean]) + + property("171 withFilter behaves like filter (2 variables)") = + val model = for + x <- Bernoulli(.5) + y <- Bernoulli(.5) + if x + yield (x,y) + val model1 = + Bernoulli(.5).flatMap { x => + Bernoulli(.5).withFilter { y => x }.map { y => + (x,y) } } + println(model.sample(N)) + false + + property("180 map after withFilter behaves like map") = false +*/ + + // property("190 sample(N).size = N") = false + // property("200 conditioning twice has the same effect as once") = false + // property("210 mean of constant sample") = false + // property("220 mean of other deterministic sample") = false + // property("230 median of constant sample") = false + // property("240 median is >= half of the elements in a sample") = false + // property("250 median is <= half of the eleemnts in a sample") = false From c535c672f4e8b4feee41c2b630514cf5e1a10c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Thu, 11 Jan 2024 11:23:31 +0100 Subject: [PATCH 02/32] radnomizeD: Remove a stale import --- src/main/scala/symsim/concrete/Randomized.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/symsim/concrete/Randomized.scala b/src/main/scala/symsim/concrete/Randomized.scala index 03d2538c..b8d9808e 100644 --- a/src/main/scala/symsim/concrete/Randomized.scala +++ b/src/main/scala/symsim/concrete/Randomized.scala @@ -2,7 +2,6 @@ package symsim.concrete import scala.annotation.targetName -import cats.data.State import org.scalacheck.{Gen, Prop} import scala.jdk.StreamConverters.* From 3a2b0e557e2fa9181224e0da2d65f031c7464b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Fri, 12 Jan 2024 12:33:35 +0100 Subject: [PATCH 03/32] IData: Make toString less eager so that we do not clog heap when printing --- src/main/scala/probula/IData.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/probula/IData.scala b/src/main/scala/probula/IData.scala index 5bddd73e..5483bf42 100644 --- a/src/main/scala/probula/IData.scala +++ b/src/main/scala/probula/IData.scala @@ -53,8 +53,11 @@ case class IData[+T](name: Name, chain: Chain[T]): def label(name: String): IData[T] = label(name.toName) + /** Show the data as a string, for the prefix of at most 100 elements + */ override def toString : String = - s"""${this.name.toString}: ${this.chain.mkString(",")}""" + val elipsis = if this.chain.drop(100).isEmpty then "" else "..." + s"""${this.name}: ${this.chain.take(100).mkString(",")} ${elipsis}""" /** Estimate the probability of an event defined by the predicate `p`. * Precondition - the sample size is finite. From 8e03c03b3ccf53e744e7f601b30e70219f9763ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Fri, 12 Jan 2024 12:33:58 +0100 Subject: [PATCH 04/32] Dist: Add uniform continous distribution --- src/main/scala/probula/Dist.scala | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/scala/probula/Dist.scala b/src/main/scala/probula/Dist.scala index f671a150..e06ec6c9 100644 --- a/src/main/scala/probula/Dist.scala +++ b/src/main/scala/probula/Dist.scala @@ -70,7 +70,7 @@ trait Dist[+T]: * * [1] Technically the resulting model does not have to be * univariate, even though it typically is. It will have the arity - * of type `U`. + * of the type `U`. * * @param name The descriptive name of the only variable in the new * model. @@ -727,3 +727,20 @@ object Uniform: def apply[T](values: T*): Uniform[T] = new Uniform[T](Name.No)(values*) + +case class UniformC(name: Name, lower: Double, upper: Double) + extends Dist[Double]: + assert (lower <= upper) + + private lazy val gen = + spire.random.Uniform[Double](lower, upper) + + def sample[S >: Double](using rng: RNG): IData[S] = + val chain = gen.toLazyList(rng) + IData(this.name, chain) + +object UniformC: + def apply(lower: Double, upper: Double): UniformC = + new UniformC(Name.No, lower, upper) + def apply(l: String, lower: Double, upper: Double): UniformC = + new UniformC(l.toName, lower, upper) From 9a06dba1eee223170d48e2195ba555bee926c78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Fri, 12 Jan 2024 20:14:38 +0100 Subject: [PATCH 05/32] build: Bump scala up to 3.3.1 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a1c4d338..3bf9dcf1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "symsim" -ThisBuild / scalaVersion := "3.1.3" +ThisBuild / scalaVersion := "3.3.1" val scalatestVersion = "3.2.14" val catsVersion = "2.6.1" From b3a4dd1a18bffd95344e38158f5ed1382576c9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 00:27:26 +0100 Subject: [PATCH 06/32] Replace Randomized with Randomized2 everywhere --- src/main/scala/probula/Dist.scala | 39 +++++-- src/main/scala/probula/IData.scala | 6 +- src/main/scala/symsim/Agent.scala | 5 - src/main/scala/symsim/ExactRL.scala | 12 +- .../concrete/BdlConcreteExpectedSarsa.scala | 4 +- .../symsim/concrete/BdlConcreteSarsa.scala | 4 +- .../symsim/concrete/ConcreteExactRL.scala | 13 ++- .../concrete/ConcreteExpectedSarsa.scala | 4 +- .../symsim/concrete/ConcreteQLearning.scala | 4 +- .../concrete/ConcreteQLearningWithDecay.scala | 4 +- .../symsim/concrete/ConcreteQTable.scala | 16 ++- .../scala/symsim/concrete/ConcreteSarsa.scala | 4 +- .../symsim/concrete/ConcreteVTable.scala | 12 +- .../scala/symsim/concrete/Randomized.scala | 5 +- .../scala/symsim/concrete/Randomized2.scala | 106 ++++++++++++++++++ .../examples/concrete/braking/Car.scala | 38 +++---- .../examples/concrete/cartpole/CartPole.scala | 42 ++++--- .../concrete/cliffwalking/CliffWalking.scala | 35 +++--- .../symsim/examples/concrete/golf/Golf.scala | 37 +++--- .../concrete/mountaincar/MountainCar.scala | 36 +++--- .../examples/concrete/pumping/Pump.scala | 60 +++++----- .../concrete/simplebandit/Bandit.scala | 28 +++-- .../examples/concrete/simplemaze/Maze.scala | 58 +++++----- .../concrete/windygrid/WindyGrid.scala | 41 +++---- .../laws/ConcreteExpectedSarsaLaws.scala | 22 ++-- .../scala/symsim/laws/ConcreteSarsaLaws.scala | 24 ++-- src/test/scala/symsim/ExperimentSpec.scala | 20 ++-- ...ConcreteExpectedSarsaIsExpectedSarsa.scala | 7 +- .../concrete/BdlConcreteSarsaIsSarsa.scala | 8 +- ...ConcreteExpectedSarsaIsExpectedSarsa.scala | 4 +- .../ConcreteQLearningIsSarsaSpec.scala | 6 +- .../concrete/ConcreteSarsaIsSarsaSpec.scala | 6 +- .../symsim/concrete/ConcreteSarsaSpec.scala | 20 ++-- .../symsim/concrete/RandomizedSpec.scala | 10 -- .../scala/symsim/concrete/UnitAgent.scala | 35 +++--- .../concrete/UnitAgentExperimentsSpec.scala | 11 +- .../concrete/braking/CarIsAgentSpec.scala | 4 +- .../examples/concrete/braking/CarSpec.scala | 36 +++--- .../concrete/braking/Experiments.scala | 13 +-- .../concrete/cartpole/Experiments.scala | 6 +- .../CliffWalkingIsAgentSpec.scala | 7 +- .../cliffwalking/CliffWalkingSpec.scala | 7 +- .../concrete/cliffwalking/Experiments.scala | 8 +- .../examples/concrete/golf/Experiments.scala | 11 +- .../concrete/golf/GolfIsAgentSpec.scala | 6 +- .../examples/concrete/golf/GolfSpec.scala | 24 ++-- .../concrete/mountaincar/Experiments.scala | 10 +- .../mountaincar/MountainCarIsAgentSpec.scala | 5 +- .../concrete/pumping/Experiments.scala | 10 +- .../concrete/pumping/PumpIsAgentSpec.scala | 6 +- .../simplebandit/BanditObjectConst.scala | 23 ++-- .../simplebandit/BanditObjectGaussian.scala | 23 ++-- .../concrete/simplemaze/BdlExperiments.scala | 7 +- .../simplemaze/ExpectedSarsaExperiments.scala | 7 +- .../concrete/simplemaze/MazeIsAgentSpec.scala | 6 +- .../concrete/simplemaze/MazeSpec.scala | 16 ++- .../simplemaze/SarsaExperiments.scala | 13 ++- .../concrete/windygrid/Experiments.scala | 8 +- .../windygrid/WindyGridIsAgentSpec.scala | 4 +- .../concrete/windygrid/WindyGridSpec.scala | 27 +++-- 60 files changed, 638 insertions(+), 435 deletions(-) create mode 100644 src/main/scala/symsim/concrete/Randomized2.scala diff --git a/src/main/scala/probula/Dist.scala b/src/main/scala/probula/Dist.scala index e06ec6c9..d9e9b2e6 100644 --- a/src/main/scala/probula/Dist.scala +++ b/src/main/scala/probula/Dist.scala @@ -52,7 +52,7 @@ trait Dist[+T]: * users of the library. They will use the variant that takes a * sample size. */ - protected def sample[U >: T](using RNG): IData[U] + def sample[U >: T](using RNG): IData[U] // Derived members @@ -728,19 +728,44 @@ object Uniform: def apply[T](values: T*): Uniform[T] = new Uniform[T](Name.No)(values*) -case class UniformC(name: Name, lower: Double, upper: Double) - extends Dist[Double]: - assert (lower <= upper) - - private lazy val gen = - spire.random.Uniform[Double](lower, upper) +/** sub class of distributions over double numbers / "real numbers" */ +trait DistD extends Dist[Double]: + + protected def gen: spire.random.Dist[Double] def sample[S >: Double](using rng: RNG): IData[S] = val chain = gen.toLazyList(rng) IData(this.name, chain) + + +case class UniformC(name: Name, lower: Double, upper: Double) + extends DistD: + assert (this.lower <= this.upper) + + protected lazy val gen = + spire.random.Uniform[Double](lower, upper) + object UniformC: def apply(lower: Double, upper: Double): UniformC = new UniformC(Name.No, lower, upper) def apply(l: String, lower: Double, upper: Double): UniformC = new UniformC(l.toName, lower, upper) + + + +case class Gaussian(name: Name, mean: Double = 0.0, stdDev: Double = 1.0) + extends DistD: + + protected lazy val gen = + spire.random.Gaussian(mean, stdDev) + +object Gaussian: + def apply(mean: Double, stdDev: Double): Gaussian = + new Gaussian(Name.No, mean, stdDev) + def apply: Gaussian = + new Gaussian(Name.No, 0.0, 1.0) + def apply(l: String, mean: Double, stdDev: Double): Gaussian = + new Gaussian(l.toName, mean, stdDev) + def apply(l: String): Gaussian = + new Gaussian(l.toName, 0.0, 1.0) diff --git a/src/main/scala/probula/IData.scala b/src/main/scala/probula/IData.scala index 5483bf42..5a73402c 100644 --- a/src/main/scala/probula/IData.scala +++ b/src/main/scala/probula/IData.scala @@ -35,7 +35,7 @@ case class IData[+T](name: Name, chain: Chain[T]): private[probula] def map[S](f: T => S): IData[S] = IData(this.name, this.chain.map(f)) - private[probula] def flatMap[S](f: T => IData[S]): IData[S] = + def flatMap[S](f: T => IData[S]): IData[S] = val newChain: Chain[S] = this.chain.map[S] { t => f(t).chain.head } this.copy (chain = newChain) @@ -100,7 +100,9 @@ extension [T](ego: IData[T]) val s = (if (N != M) ego.chain.tail else ego.chain).sorted s(M / 2) - /** Compute a mean for a numeric sample. */ + /** Compute a mean for a numeric sample. + * Note: This fails for large samples due to overflow + */ def mean(using math.Numeric[T]): Double = ego.chain.sum.toDouble / ego.size diff --git a/src/main/scala/symsim/Agent.scala b/src/main/scala/symsim/Agent.scala index c4f17354..3d8080a3 100644 --- a/src/main/scala/symsim/Agent.scala +++ b/src/main/scala/symsim/Agent.scala @@ -87,11 +87,6 @@ trait AgentConstraints[State, ObservableState, Action, Reward, Scheduler[_]]: */ given schedulerIsMonad: Monad[Scheduler] - /** We need to be able to iterate over a schedule (fold) to sequence and merge - * schedule elements. - */ - given schedulerIsFoldable: Foldable[Scheduler] - /** We need to be able to run tests on scheduled values. */ given canTestInScheduler: CanTestIn[Scheduler] diff --git a/src/main/scala/symsim/ExactRL.scala b/src/main/scala/symsim/ExactRL.scala index 3995e798..03d0feec 100644 --- a/src/main/scala/symsim/ExactRL.scala +++ b/src/main/scala/symsim/ExactRL.scala @@ -76,15 +76,11 @@ trait ExactRL[State, ObservableState, Action, Reward, Scheduler[_]] yield (fin._1, qL_tt, decay (ε)) - /** Executes as many full learning episodes (until the final state of agent is - * reached) as the given state scheduler generates. For this method to work - * the scheduler needs to be foldable, and we use foldRight with Eval, to - * make the evaluation lazy. We force the evaluation when we - * are done to return the value. However, to my best understanding, if the - * Scheduler is lazy then the evaluation is not really doing more than just - * formulating the thunk of that scheduler. + /** Executes as many full learning episodes (until the final state of agent + * is reached) as the given state LazyList generates. The list should be + * finite. We force the evaluation when we are done to return the value. */ - final def learn (f: VF, q_l: List[VF], ss: => Scheduler[State]) + final def learn (f: VF, q_l: List[VF], ss: LazyList[State]) : Scheduler[(VF, List[VF])] = val result = ss.foldM[Scheduler, (VF, List[VF], Probability)] (f, q_l, ε0) (learningEpisode) diff --git a/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala b/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala index f0ddd4e9..3a175411 100644 --- a/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala +++ b/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala @@ -8,12 +8,12 @@ case class BdlConcreteExpectedSarsa [ ObservableState: BoundedEnumerable, Action: BoundedEnumerable ] ( - val agent: Agent[State, ObservableState, Action, Double, Randomized], + val agent: Agent[State, ObservableState, Action, Double, Randomized2], val alpha: Double, val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends BdlLearn[State, ObservableState, Action, Double, Randomized], +) extends BdlLearn[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala b/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala index 7e05c9ba..1fbad143 100644 --- a/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala +++ b/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala @@ -8,12 +8,12 @@ case class BdlConcreteSarsa [ ObservableState: BoundedEnumerable, Action: BoundedEnumerable ] ( - val agent: Agent[State, ObservableState, Action, Double, Randomized], + val agent: Agent[State, ObservableState, Action, Double, Randomized2], val alpha: Double, val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends BdlLearn[State, ObservableState, Action, Double, Randomized], +) extends BdlLearn[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteExactRL.scala b/src/main/scala/symsim/concrete/ConcreteExactRL.scala index b66d2f29..a75e4847 100644 --- a/src/main/scala/symsim/concrete/ConcreteExactRL.scala +++ b/src/main/scala/symsim/concrete/ConcreteExactRL.scala @@ -3,8 +3,11 @@ package concrete import cats.kernel.BoundedEnumerable +given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply + trait ConcreteExactRL[State, ObservableState, Action] - extends ExactRL[State, ObservableState, Action, Double, Randomized]: + extends ExactRL[State, ObservableState, Action, Double, Randomized2]: import agent.instances.* import agent.instances.given @@ -23,9 +26,9 @@ trait ConcreteExactRL[State, ObservableState, Action] // symbolic or approximate algos we should promote this to the trait def runQ: (Q, List[Q]) = - val initials = Randomized.repeat (agent.initialize).take (episodes) - val schedule = learn (vf.initialize, List[VF](), initials) - (schedule.head._1, schedule.head._2) + val initials = agent.initialize.sample(episodes) + val outcome = learn (vf.initialize, List[VF](), initials).sample() + (outcome._1, outcome._2) override def run: Policy = qToPolicy (this.runQ._1) @@ -43,5 +46,5 @@ trait ConcreteExactRL[State, ObservableState, Action] * the inner distribution is over randomness in the learning * process/environment. */ - def evaluate (p: Policy): Randomized[Randomized[Double]] = + def evaluate (p: Policy): Randomized2[Randomized2[Double]] = evaluate (p, agent.initialize) diff --git a/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala b/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala index 8fce6b59..579626ad 100644 --- a/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala +++ b/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala @@ -2,12 +2,12 @@ package symsim package concrete case class ConcreteExpectedSarsa[State, ObservableState, Action] ( - val agent: Agent[State, ObservableState, Action, Double, Randomized], + val agent: Agent[State, ObservableState, Action, Double, Randomized2], val alpha: Double, val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends ExpectedSarsa[State, ObservableState, Action, Double, Randomized], +) extends ExpectedSarsa[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteQLearning.scala b/src/main/scala/symsim/concrete/ConcreteQLearning.scala index c662544c..ed9c722a 100644 --- a/src/main/scala/symsim/concrete/ConcreteQLearning.scala +++ b/src/main/scala/symsim/concrete/ConcreteQLearning.scala @@ -8,12 +8,12 @@ case class ConcreteQLearning [ ObservableState: BoundedEnumerable, Action: BoundedEnumerable ] ( - val agent: Agent[State, ObservableState, Action, Double, Randomized], + val agent: Agent[State, ObservableState, Action, Double, Randomized2], val alpha: Double, val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends QLearning[State, ObservableState, Action, Double, Randomized], +) extends QLearning[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala b/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala index d33c56b5..34c1707a 100644 --- a/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala +++ b/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala @@ -9,13 +9,13 @@ case class ConcreteQLearningWithDecay [ Action: BoundedEnumerable ] ( - val agent: Agent[State, ObservableState, Action, Double, Randomized], + val agent: Agent[State, ObservableState, Action, Double, Randomized2], val alpha: Double, val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends QLearning[State, ObservableState, Action, Double, Randomized], +) extends QLearning[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], BoundedEpsilonDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteQTable.scala b/src/main/scala/symsim/concrete/ConcreteQTable.scala index 79ae3a51..410c7b75 100644 --- a/src/main/scala/symsim/concrete/ConcreteQTable.scala +++ b/src/main/scala/symsim/concrete/ConcreteQTable.scala @@ -2,13 +2,17 @@ package symsim package concrete import cats.kernel.BoundedEnumerable -import cats.syntax.option.* +import cats.syntax.all.* import org.typelevel.paiges.Doc import symsim.QTable +import symsim.concrete.Randomized2.given + +// import cats.Monad.nonInheritedOps.toFlatMapOps + class ConcreteQTable[ObservableState: BoundedEnumerable, Action: BoundedEnumerable] - extends QTable[ObservableState, Action, Double, Randomized]: + extends QTable[ObservableState, Action, Double, Randomized2]: def value (q: Q) (s: ObservableState, a: Action): Double = q (s, a) @@ -27,11 +31,11 @@ class ConcreteQTable[ObservableState: BoundedEnumerable, Action: BoundedEnumerab def chooseAction (ε: Probability) (q: Q) (s: ObservableState) - : Randomized[Action] = for - explore <- Randomized.coin (ε) + : Randomized2[Action] = for + explore <- Randomized2.coin (ε) action <- if explore - then Randomized.oneOf (allActions*) - else Randomized.const (bestAction (q) (s)) + then Randomized2.oneOf (allActions*) + else Randomized2.const (bestAction (q) (s)) yield action diff --git a/src/main/scala/symsim/concrete/ConcreteSarsa.scala b/src/main/scala/symsim/concrete/ConcreteSarsa.scala index 709f890b..7afc355f 100644 --- a/src/main/scala/symsim/concrete/ConcreteSarsa.scala +++ b/src/main/scala/symsim/concrete/ConcreteSarsa.scala @@ -8,12 +8,12 @@ case class ConcreteSarsa [ ObservableState: BoundedEnumerable, Action: BoundedEnumerable ] ( - val agent: Agent[State, ObservableState, Action, Double, Randomized], + val agent: Agent[State, ObservableState, Action, Double, Randomized2], val alpha: Double, val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends Sarsa[State, ObservableState, Action, Double, Randomized], +) extends Sarsa[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteVTable.scala b/src/main/scala/symsim/concrete/ConcreteVTable.scala index f2014e3b..d4428b44 100644 --- a/src/main/scala/symsim/concrete/ConcreteVTable.scala +++ b/src/main/scala/symsim/concrete/ConcreteVTable.scala @@ -1,22 +1,24 @@ package symsim package concrete +import cats.syntax.all.* + trait ConcreteVTable[State, ObservableState, Action] - extends VTable[State, ObservableState, Action, Double, Randomized]: + extends VTable[State, ObservableState, Action, Double, Randomized2]: this: ConcreteExactRL[State, ObservableState, Action] => import agent.instances.* - type Scheduler[A] = Randomized[A] + type Scheduler[A] = Randomized2[A] /** TODO: is this function used by VTable algorithms at all? * I have replaced it by unimplemented for now, for safety. */ def bestAction (v: V) (s: State): Action = ??? def chooseAction (v: V) (s: State): Scheduler[Action] = for - explore <- Randomized.coin (this.epsilon0) + explore <- Randomized2.coin (this.epsilon0) action <- if explore - then Randomized.oneOf (allActions*) - else Randomized.const (bestAction (v) (s)) + then Randomized2.oneOf (allActions*) + else Randomized2.const (bestAction (v) (s)) yield action diff --git a/src/main/scala/symsim/concrete/Randomized.scala b/src/main/scala/symsim/concrete/Randomized.scala index b8d9808e..71fbfe42 100644 --- a/src/main/scala/symsim/concrete/Randomized.scala +++ b/src/main/scala/symsim/concrete/Randomized.scala @@ -75,9 +75,6 @@ object Randomized: given randomizedIsMonad: cats.Monad[Randomized] = cats.instances.lazyList.catsStdInstancesForLazyList - given randomizedIsFoldable: cats.Foldable[Randomized] = - cats.instances.lazyList.catsStdInstancesForLazyList - given canTestInRandomized: symsim.CanTestIn[Randomized] = new symsim.CanTestIn[Randomized] { @@ -116,7 +113,7 @@ object Randomized: * not have n samples. * */ - def sample (n: Int): List[A] = self.take(n).toList + def sample (n: Int): LazyList[A] = self.take(n) /** Perform an imperative operation that depends on one sample from this * Randomized. This is mostly meant for IO at this point. diff --git a/src/main/scala/symsim/concrete/Randomized2.scala b/src/main/scala/symsim/concrete/Randomized2.scala new file mode 100644 index 00000000..01fdc28f --- /dev/null +++ b/src/main/scala/symsim/concrete/Randomized2.scala @@ -0,0 +1,106 @@ +package symsim.concrete + +import scala.annotation.targetName +import org.scalacheck.{Gen, Prop} +import probula.{Dist, RNG} + +opaque type Randomized2[+A] = Dist[A] + +object Randomized2: + + /** Create a generator that produces a single 'a'. Used to create + * deterministic values when a scheduler/randomized type is + * expected. + */ + def const[A] (a: =>A): Randomized2[A] = + probula.Dirac (a) + + def prob: Randomized2[Double] = + probula.UniformC(0.0, 1.0) + + def between (minInclusive: Int, maxExclusive: Int): Randomized2[Int] = + probula.Uniform(minInclusive, maxExclusive+1) + + def between (minInclusive: Double, maxExclusive: Double): Randomized2[Double] = + probula.UniformC(minInclusive, maxExclusive) + + def gaussian (mean: Double = 0.0, stdDev: Double = 1.0): Randomized2[Double] = + probula.Gaussian(mean, stdDev) + + def coin (bias: Probability): Randomized2[Boolean] = + probula.Bernoulli(bias) + + def oneOf[A] (choices: A*): Randomized2[A] = + probula.Uniform(choices*) + + /** For this representation of Randomized, repeat does nothing (an identity), + * we should probably remove repeat from the API. */ + def repeat[A] (ra: =>Randomized2[A]): Randomized2[A] = ra + + + given randomizedIsMonad: cats.Monad[Randomized2] = new cats.Monad[Randomized2]: + def flatMap[A, B](fa: Randomized2[A])(f: A => Randomized2[B]): Randomized2[B] = + fa.flatMap(f) + def pure[A](x: A): Randomized2[A] = + probula.Dirac[A] (x) + + import probula.{Name, IData, RNG} + + def tailRecM[A, B](ini: A)(f: A => Randomized2[Either[A, B]]): Randomized2[B] = + val a0 = f(ini) + new Dist[B]: + def name: Name = Name.Suffixed(a0.name, "tailRecM") + def sample[C >: B](using RNG): IData[C] = + val newChain = summon[cats.Monad[LazyList]] + .tailRecM[A, B](ini) { a => f(a).sample.chain } + IData(name, newChain) + + given canTestInRandomized (using RNG): symsim.CanTestIn[Randomized2] = + new symsim.CanTestIn[Randomized2] { + + @targetName ("toPropBoolean") + def toProp (rProp: Randomized2[Boolean]) = + Prop.forAllNoShrink (toGen (rProp)) (identity[Boolean]) + + // This is a nasty hack that costs as a lot on memory in tests (but + // probably not in experiments). Unfortunately, I do not see an easy way + // to add a completely new generator for scalacheck that encapsulates + // Randomized. The Gen class appears to be sealed and pimping cannot add + // state to objects? + def toGen[A] (ra: => Randomized2[A]): Gen[A] = + val list: LazyList[A] = ra.sample.chain + Gen.choose(0, 1000) + .map { i => list (i) } + } + + + /** This extensions should ideally be used at a top-level of the program, + * as they loose the type annotation for the side effect of randomness. + */ + extension [A] (self: Randomized2[A]) + /** Get one sample from randomized. Note that the sample will be random but + * always the same if you call several times. + * (at least in the current implementation) + */ + def sample () (using RNG): A = + self.sample(1)(using summon[RNG]).chain.head + + /** Get n samples from randomized. Note that the sample will be random but + * always the same if you call several times. + * (at least in the current implementation) + * + * Note: as long as we have not refactored the underlying implementation + * of Randomized, this is unsafe. It is possible that randomized does + * not have n samples. + * + */ + def sample (n: Int) (using RNG): LazyList[A] = + self.sample(n).chain.take(n) + + /** Perform an imperative operation that depends on one sample from this + * Randomized. This is mostly meant for IO at this point. + */ + def run (f: A => Unit): Unit = f(self.sample ()) + + def filter (p: A => Boolean): Randomized2[A] = + self.filter (p) diff --git a/src/main/scala/symsim/examples/concrete/braking/Car.scala b/src/main/scala/symsim/examples/concrete/braking/Car.scala index 79b8aaae..c9012137 100644 --- a/src/main/scala/symsim/examples/concrete/braking/Car.scala +++ b/src/main/scala/symsim/examples/concrete/braking/Car.scala @@ -1,7 +1,8 @@ package symsim package examples.concrete.braking -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 +import cats.syntax.all.* /** * We map the car states to @@ -19,8 +20,8 @@ type CarObservableState = CarState type CarAction = Double type CarReward = Double -object Car - extends Agent[CarState, CarObservableState, CarAction, CarReward, Randomized] +class Car (using probula.RNG) + extends Agent[CarState, CarObservableState, CarAction, CarReward, Randomized2] with Episodic: /** The episode should be guaranteed to terminate after @@ -34,7 +35,7 @@ object Car private val t: Double = 2.0 /** Evidence of type class membership for this agent. */ - val instances = CarInstances + val instances = new CarInstances def isFinal (s: CarState): Boolean = s.v == 0.0 || s.p >= 10 || Math.abs (s.p) >= 1000.0 @@ -57,7 +58,7 @@ object Car // TODO: this is now deterministic but eventually needs to be randomized - def step (s: CarState) (a: CarAction): Randomized[(CarState, CarReward)] = + def step (s: CarState) (a: CarAction): Randomized2[(CarState, CarReward)] = require (instances.enumAction.membersAscending.contains (a)) // Stop moving when velecity is zero, braking is not moving backwards val t1 = Math.min (- s.v / a, t) @@ -65,14 +66,14 @@ object Car val v1 = if p1 == 10 then 0 else Math.max(s.v + a * t1, 0.0) val s1 = CarState (v = v1, p = p1) - Randomized.const (s1, carReward (s1) (a)) + Randomized2.const (s1, carReward (s1) (a)) - def initialize: Randomized[CarState] = for - v <- Randomized.repeat (Randomized.between (0.0, 15.0)) - p <- Randomized.between (0.0, 20.0) - s = CarState (v, p) if !isFinal (s) - yield s + def initialize: Randomized2[CarState] = (for + v <- Randomized2.between (0.0, 15.0) + p <- Randomized2.between (0.0, 20.0) + s = CarState (v, p) + yield s).filter { !this.isFinal (_)} end Car @@ -80,8 +81,8 @@ end Car /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object CarInstances - extends AgentConstraints[CarState, CarObservableState, CarAction, CarReward, Randomized]: +class CarInstances (using probula.RNG) + extends AgentConstraints[CarState, CarObservableState, CarAction, CarReward, Randomized2]: import cats.{Eq, Monad, Foldable} import cats.kernel.BoundedEnumerable @@ -100,14 +101,11 @@ object CarInstances yield CarState (v,p) BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = - concrete.Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + concrete.Randomized2.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = - concrete.Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = - concrete.Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + concrete.Randomized2.canTestInRandomized lazy val genCarState: Gen[CarState] = for v <- Gen.choose (0.0, 10.0) diff --git a/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala b/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala index 057693dc..1bf97fb3 100644 --- a/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala +++ b/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala @@ -1,7 +1,8 @@ package symsim package examples.concrete.cartpole -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 +import cats.syntax.all.* /** The state of the CartPole, an abstract type shared by the continuous state @@ -84,9 +85,9 @@ val PvCutPoints = * Correct equations for the dynamics of the cart-pole system. * https://coneural.org/florian/papers/05_cart_pole.pdf */ -object CartPole +class CartPole (using probula.RNG) extends Agent[CartPoleState, CartPoleObservableState, CartPoleAction, - CartPoleReward, Randomized], + CartPoleReward, Randomized2], Episodic: /** The episode guarantees to terminate after TimeHorizon steps. @@ -121,7 +122,7 @@ object CartPole : CartPoleReward = 1.0 def step (s: CartPoleState) (a: CartPoleAction) - : Randomized[(CartPoleState, CartPoleReward)] = + : Randomized2[(CartPoleState, CartPoleReward)] = require (instances.enumAction.membersAscending.contains (a)) val cos_τ = Math.cos (s.pa) val sin_τ = Math.sin (s.pa) @@ -140,18 +141,18 @@ object CartPole val pa1 = s.pa + τ * s.pv val pv1 = s.pv + τ * τAcc val s1 = CartPoleState (cp1, cv1, pa1, pv1) - Randomized.const (s1, cartPoleReward (s1) (a)) + Randomized2.const (s1, cartPoleReward (s1) (a)) - def initialize: Randomized[CartPoleState] = for - cp <- Randomized.repeat (Randomized.between (-0.05, 0.05)) - cv <- Randomized.repeat (Randomized.between (-0.05, 0.05)) - pa <- Randomized.repeat (Randomized.between (-0.05, 0.05)) - pv <- Randomized.between(-0.05, 0.05) - s = CartPoleState (cp, cv, pa, pv) if !isFinal (s) - yield s + def initialize: Randomized2[CartPoleState] = (for + cp <- Randomized2.between (-0.05, 0.05) + cv <- Randomized2.between (-0.05, 0.05) + pa <- Randomized2.between (-0.05, 0.05) + pv <- Randomized2.between (-0.05, 0.05) + s = CartPoleState (cp, cv, pa, pv) + yield s).filter { !this.isFinal (_) } /** Evidence of type class membership for this agent. */ - val instances = CartPoleInstances + val instances = new CartPoleInstances end CartPole @@ -159,9 +160,9 @@ end CartPole /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object CartPoleInstances +class CartPoleInstances (using probula.RNG) extends AgentConstraints[CartPoleState, CartPoleObservableState, - CartPoleAction, CartPoleReward, Randomized]: + CartPoleAction, CartPoleReward, Randomized2]: import cats.{Eq, Monad, Foldable} import cats.kernel.BoundedEnumerable @@ -179,14 +180,11 @@ object CartPoleInstances yield CartPoleObservableState (cp, cv, pa, pv) BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = - concrete.Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + concrete.Randomized2.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = - concrete.Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = - concrete.Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + concrete.Randomized2.canTestInRandomized lazy val genCartPoleState: Gen[CartPoleState] = for cp <- Gen.choose[Double] (CpMin, CpMax) diff --git a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala index c8771874..1f1c08b0 100644 --- a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala +++ b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala @@ -3,10 +3,11 @@ package examples.concrete.cliffWalking import cats.{Eq, Foldable, Monad} import cats.kernel.BoundedEnumerable +import cats.syntax.all.* import org.scalacheck.{Arbitrary, Gen} -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 case class CWState (x: Int, y: Int): override def toString: String = s"($x,$y)" @@ -26,8 +27,8 @@ enum CWAction: val BoardWidth: Int = 11 val BoardHeight: Int = 3 -object CliffWalking - extends Agent[CWState, CWObservableState, CWAction, CWReward, Randomized] +class CliffWalking (using probula.RNG) + extends Agent[CWState, CWObservableState, CWAction, CWReward, Randomized2] with Episodic: /** The episode should be guaranteed to terminate after @@ -61,25 +62,25 @@ object CliffWalking else -1.0 - def step (s: CWState) (a: CWAction): Randomized[(CWState, CWReward)] = + def step (s: CWState) (a: CWAction): Randomized2[(CWState, CWReward)] = val s1 = move (s, a) - Randomized.const (s1, cwReward (s1) (a)) + Randomized2.const (s1, cwReward (s1) (a)) - def initialize: Randomized[CWState] = for - x <- Randomized.repeat (Randomized.between (0, BoardWidth)) - y <- Randomized.between (0, BoardHeight) - s = CWState (x, y) if !isFinal (s) - yield s + def initialize: Randomized2[CWState] = { for + x <- Randomized2.between (0, BoardWidth) + y <- Randomized2.between (0, BoardHeight) + s = CWState (x, y) + yield s }.filter { !this.isFinal (_) } - val instances = CliffWalkingInstances + val instances = new CliffWalkingInstances end CliffWalking /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object CliffWalkingInstances - extends AgentConstraints[CWState, CWObservableState, CWAction, CWReward, Randomized]: +class CliffWalkingInstances (using probula.RNG) + extends AgentConstraints[CWState, CWObservableState, CWAction, CWReward, Randomized2]: given enumAction: BoundedEnumerable[CWAction] = BoundedEnumerableFromList (CWAction.Up, CWAction.Down, CWAction.Left, CWAction.Right) @@ -92,11 +93,11 @@ object CliffWalkingInstances BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad - val schedulerIsFoldable: Foldable[Randomized] = Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized lazy val genCWState: Gen[CWState] = for x <- Gen.oneOf ((0 to BoardWidth).toSeq) diff --git a/src/main/scala/symsim/examples/concrete/golf/Golf.scala b/src/main/scala/symsim/examples/concrete/golf/Golf.scala index f1f103eb..6298d127 100644 --- a/src/main/scala/symsim/examples/concrete/golf/Golf.scala +++ b/src/main/scala/symsim/examples/concrete/golf/Golf.scala @@ -3,10 +3,11 @@ package examples.concrete.golf import cats.{Eq, Foldable, Monad} import cats.kernel.BoundedEnumerable +import cats.syntax.all.* import org.scalacheck.{Arbitrary, Gen} -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 /** * Russell, Norvig, Example 3.6 p. 61 [83] and the following pages. @@ -35,8 +36,8 @@ enum Direction: type GolfAction = (Club, Direction) -object Golf extends - Agent[GolfState, GolfObservableState, GolfAction, GolfReward, Randomized], +class Golf (using probula.RNG) extends + Agent[GolfState, GolfObservableState, GolfAction, GolfReward, Randomized2], Episodic: val TimeHorizon: Int = 2000 @@ -92,13 +93,13 @@ object Golf extends def valid (s: GolfState): Boolean = s >= 1 && s <= 10 - def step (s: GolfState) (a: GolfAction): Randomized[(GolfState, GolfReward)] = - Randomized.const(successor (s) (a), golfReward (s) (a)) + def step (s: GolfState) (a: GolfAction): Randomized2[(GolfState, GolfReward)] = + Randomized2.const(successor (s) (a), golfReward (s) (a)) - def initialize: Randomized[GolfState] = - Randomized.const (StartState) + def initialize: Randomized2[GolfState] = + Randomized2.const (StartState) - val instances = GolfInstances + val instances = new GolfInstances end Golf @@ -106,9 +107,9 @@ end Golf /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object GolfInstances +class GolfInstances (using probula.RNG) extends AgentConstraints[GolfState, GolfObservableState, GolfAction, - GolfReward, Randomized]: + GolfReward, Randomized2]: given enumAction: BoundedEnumerable[GolfAction] = BoundedEnumerableFromList ((Club.P, Direction.L), (Club.P, Direction.R), @@ -117,21 +118,21 @@ object GolfInstances given enumState: BoundedEnumerable[GolfObservableState] = BoundedEnumerableFromList (1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = - Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = - Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized lazy val genGolfState: Gen[GolfState] = Gen.choose (1, 10) - given arbitraryState: Arbitrary[GolfState] = Arbitrary (genGolfState) + given arbitraryState: Arbitrary[GolfState] = + Arbitrary (genGolfState) given eqGolfState: Eq[GolfState] = Eq.fromUniversalEquals - given arbitraryReward: Arbitrary[GolfReward] = Arbitrary (Gen.double) + given arbitraryReward: Arbitrary[GolfReward] = + Arbitrary (Gen.double) given rewardArith: Arith[GolfReward] = Arith.arithDouble diff --git a/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala b/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala index b0614099..aacb3ade 100644 --- a/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala +++ b/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala @@ -2,13 +2,16 @@ package symsim package examples.concrete.mountaincar import Math.cos +import probula.RNG import cats.{Eq, Foldable, Monad} import cats.kernel.BoundedEnumerable +import cats.syntax.all.* import org.scalacheck.{Arbitrary, Gen} -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 +import symsim.concrete.Randomized2.given /** * We map the car states to @@ -27,8 +30,8 @@ type CarAction = Double type CarReward = Double -object MountainCar - extends Agent[CarState, CarObservableState, CarAction, CarReward, Randomized] +class MountainCar (using RNG) + extends Agent[CarState, CarObservableState, CarAction, CarReward, Randomized2] with Episodic: val TimeHorizon: Int = 3000000 @@ -65,23 +68,22 @@ object MountainCar // TODO: this is now deterministic but eventually needs to be randomized - def step (s: CarState) (a: CarAction): Randomized[(CarState, CarReward)] = + def step (s: CarState) (a: CarAction): Randomized2[(CarState, CarReward)] = val v = s.v + (gravity * mass * cos (3.0 * s.p) + a / mass - friction * s.v) * dt val p = s.p + (v * dt) val (v1, p1) = if p < -1.2 then (-1.2, 0.0) else (v, p) val s1 = CarState (v = v1.max (-1.5).min (1.5), p = p1.min (0.5)) - Randomized.const (s1 -> carReward (s1) (a)) + Randomized2.const (s1 -> carReward (s1) (a)) - def initialize: Randomized[CarState] = for - p <- Randomized.repeat (Randomized.between (-1.2, 0.5)) - v <- Randomized.repeat (Randomized.between (-1.5, 1.5)) + def initialize: Randomized2[CarState] = (for + p <- Randomized2.between (-1.2, 0.5) + v <- Randomized2.between (-1.5, 1.5) s = CarState (v=v, p=p) - if !isFinal (s) - yield s + yield s).filter { !this.isFinal (_) } - val instances = MountainCarInstances + val instances = new MountainCarInstances end MountainCar @@ -89,8 +91,8 @@ end MountainCar /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object MountainCarInstances - extends AgentConstraints[CarState, CarObservableState, CarAction, CarReward, Randomized]: +class MountainCarInstances (using RNG) + extends AgentConstraints[CarState, CarObservableState, CarAction, CarReward, Randomized2]: given enumAction: BoundedEnumerable[CarAction] = BoundedEnumerableFromList (-0.2, 0.0, 0.2) @@ -103,11 +105,11 @@ object MountainCarInstances BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized lazy val genCarState: Gen[CarState] = for p <- Gen.choose (-1.2, 0.5) diff --git a/src/main/scala/symsim/examples/concrete/pumping/Pump.scala b/src/main/scala/symsim/examples/concrete/pumping/Pump.scala index c9a2b072..e89f86d8 100644 --- a/src/main/scala/symsim/examples/concrete/pumping/Pump.scala +++ b/src/main/scala/symsim/examples/concrete/pumping/Pump.scala @@ -7,7 +7,8 @@ package examples.concrete.pumping * August 2021. (esp. Chapters 2 and 3) */ -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 +import cats.syntax.all.* /** Constants as fit in the model of AHHP (reference above). @@ -93,8 +94,8 @@ type PumpAction = Double type PumpReward = Double -object Pump extends - Agent[PumpState, ObservablePumpState, PumpAction, PumpReward, Randomized], +class Pump (using probula.RNG) extends + Agent[PumpState, ObservablePumpState, PumpAction, PumpReward, Randomized2], Episodic: /** An upper bound on episode duration. Used only in testing */ @@ -132,17 +133,17 @@ object Pump extends override def step (s: PumpState) (a: PumpAction) - : Randomized[(PumpState, PumpReward)] = + : Randomized2[(PumpState, PumpReward)] = require (instances.enumAction.membersAscending.contains (a)) for - nf <- Randomized.gaussian (0.0, 1.0) + nf <- Randomized2.gaussian (0.0, 1.0) f1 = a + nf - nd <- Randomized.gaussian (0.1, 0.01) + nd <- Randomized2.gaussian (0.1, 0.01) cd <- getDemand (s.t%24 + 1) d = cd + nd tl1 = s.tl + c1 * (f1 - d) h1 = s.h + c4 * (s.w + (c1*f1 / Math.PI)) - nw <- Randomized.gaussian (0.0, 1.0) + nw <- Randomized2.gaussian (0.0, 1.0) w1 = s.w - c2*(c1*f1) + c3 + (amp*Math.sin (2*Math.PI*(s.t + phase) / freq)) + nw hm1 = (1.0 / k)*s.phm.sum @@ -152,26 +153,26 @@ object Pump extends yield (s1, pr) - def getDemand (t: Int): Randomized[Double] = + def getDemand (t: Int): Randomized2[Double] = require (t >= 0 && t <= 24) - if t < 5 then Randomized.between (5.0, 15.0) - if t < 12 then Randomized.between (15.0, 45.0) - if t < 22 then Randomized.between (20.0, 38.0) - else Randomized.between (5.0, 20.0) - - - def initialize: Randomized[PumpState] = for - f <- Randomized.const (80) - h <- Randomized.const (10.0) - hm <- Randomized.const (10) - tl <- Randomized.const (1000) - w <- Randomized.const (9.11) - phm <- Randomized.const (List (10.0, 10.0, 10.0, 10.0, 10.0)) + if t < 5 then Randomized2.between (5.0, 15.0) + if t < 12 then Randomized2.between (15.0, 45.0) + if t < 22 then Randomized2.between (20.0, 38.0) + else Randomized2.between (5.0, 20.0) + + + def initialize: Randomized2[PumpState] = for + f <- Randomized2.const (80) + h <- Randomized2.const (10.0) + hm <- Randomized2.const (10) + tl <- Randomized2.const (1000) + w <- Randomized2.const (9.11) + phm <- Randomized2.const (List (10.0, 10.0, 10.0, 10.0, 10.0)) s = PumpState (f, h, hm, tl, 0, w, phm) _ = assert (!isFinal (s)) yield s - val instances = PumpInstances + val instances = new PumpInstances end Pump @@ -179,9 +180,9 @@ end Pump /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object PumpInstances +class PumpInstances (using probula.RNG) extends AgentConstraints[PumpState, ObservablePumpState, PumpAction, - PumpReward, Randomized]: + PumpReward, Randomized2]: import cats.{Eq, Monad, Foldable} import cats.kernel.BoundedEnumerable @@ -203,14 +204,11 @@ object PumpInstances yield ObservablePumpState (f, h, hm, tl) BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = - concrete.Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + concrete.Randomized2.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = - concrete.Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = - concrete.Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + concrete.Randomized2.canTestInRandomized lazy val genPumpState: Gen[PumpState] = for f <- Gen.choose[Double] (FLOW_MIN, FLOW_MAX) diff --git a/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala b/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala index 80af2949..89a836ce 100644 --- a/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala +++ b/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala @@ -3,10 +3,11 @@ package examples.concrete.simplebandit import cats.{Eq, Foldable, Monad} import cats.kernel.BoundedEnumerable +import cats.syntax.all.* import org.scalacheck.{Arbitrary, Gen} -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 /** * Sutton, Barto, Section 2.5 p 32. @@ -29,8 +30,8 @@ type BanditReward = Double type BanditAction = Int -class Bandit (banditReward: List [Randomized[BanditReward]]) - extends Agent[BanditState, BanditState, BanditAction, BanditReward, Randomized] +class Bandit (banditReward: List [Randomized2[BanditReward]]) (using probula.RNG) + extends Agent[BanditState, BanditState, BanditAction, BanditReward, Randomized2] with Episodic: val TimeHorizon: Int = 3 @@ -41,21 +42,21 @@ class Bandit (banditReward: List [Randomized[BanditReward]]) // Bandit is discrete override def observe (s: BanditState): BanditState = s - override def step (s: BanditState) (a: BanditAction): Randomized[(BanditState, BanditReward)] = + override def step (s: BanditState) (a: BanditAction): Randomized2[(BanditState, BanditReward)] = for r <- banditReward (a) yield (true, r) - override def initialize: Randomized[BanditState] = - Randomized.const (false) + override def initialize: Randomized2[BanditState] = + Randomized2.const (false) - override val instances = BanditInstances (banditReward) + override val instances = new BanditInstances (banditReward) end Bandit /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -class BanditInstances (banditReward: List [Randomized[BanditReward]]) - extends AgentConstraints[BanditState, BanditState, BanditAction, BanditReward, Randomized]: +class BanditInstances (banditReward: List [Randomized2[BanditReward]]) (using probula.RNG) + extends AgentConstraints[BanditState, BanditState, BanditAction, BanditReward, Randomized2]: given enumAction: BoundedEnumerable[BanditAction] = BoundedEnumerableFromList (List.range(0, banditReward.size)*) @@ -63,13 +64,10 @@ class BanditInstances (banditReward: List [Randomized[BanditReward]]) given enumState: BoundedEnumerable[BanditState] = BoundedEnumerableFromList (false, true) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = Randomized2.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = - Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = - Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized lazy val genBanditState: Gen[BanditState] = Gen.oneOf (false, true) diff --git a/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala b/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala index a29a4161..af4732cc 100644 --- a/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala +++ b/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala @@ -3,10 +3,11 @@ package examples.concrete.simplemaze import cats.{Eq, Foldable, Monad} import cats.kernel.BoundedEnumerable +import cats.syntax.all.* import org.scalacheck.{Arbitrary, Gen} -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 /** * Russell, Norvig, Fig 17.1, p. 646 @@ -39,16 +40,16 @@ type MazeState = (Int, Int, Int) type MazeObservableState = (Int, Int) type MazeReward = Double -sealed trait MazeAction -case object Left extends MazeAction -case object Right extends MazeAction -case object Up extends MazeAction -case object Down extends MazeAction +def valid (s: MazeState): Boolean = + s._1 >= 1 && s._1 <= 4 && s._2 >= 1 && s._2 <= 3 && (s._1, s._2) != (2, 2) +enum MazeAction: + case Left, Right, Up, Down +import MazeAction.* -object Maze +class Maze (using probula.RNG) extends - Agent[MazeState, MazeObservableState, MazeAction, MazeReward, Randomized], + Agent[MazeState, MazeObservableState, MazeAction, MazeReward, Randomized2], Episodic: val TimeHorizon: Int = 2000 @@ -67,9 +68,9 @@ object Maze case (_, _) => -1.0 - def distort (a: MazeAction): Randomized[MazeAction] = a match - case Up | Down => Randomized.oneOf (Left, Right) - case Left | Right => Randomized.oneOf (Up, Down) + def distort (a: MazeAction): Randomized2[MazeAction] = a match + case Up | Down => Randomized2.oneOf (Left, Right) + case Left | Right => Randomized2.oneOf (Up, Down) def successor (s: MazeState) (a: MazeAction): MazeState = @@ -81,26 +82,23 @@ object Maze case Right => (s._1+1, s._2, s._3+1) if valid (result) then result else s - def valid (s: MazeState): Boolean = - s._1 >= 1 && s._1 <= 4 && s._2 >= 1 && s._2 <= 3 && (s._1, s._2) != (2, 2) - val attention = 0.8 - def step (s: MazeState) (a: MazeAction): Randomized[(MazeState, MazeReward)] = + def step (s: MazeState) (a: MazeAction): Randomized2[(MazeState, MazeReward)] = for - precise <- Randomized.coin (attention) - action <- if precise then Randomized.const (a) else distort (a) + precise <- Randomized2.coin (attention) + action <- if precise then Randomized2.const (a) else distort (a) newState = successor (s) (action) yield (newState, mazeReward (newState)) - def initialize: Randomized[MazeState] = for - x <- Randomized.repeat(Randomized.between(1, 4)) - y <- Randomized.between(1, 3) + def initialize: Randomized2[MazeState] = { for + x <- Randomized2.between (1, 4) + y <- Randomized2.between (1, 3) t = 0 - s = (x, y, t) if !isFinal(s) && valid (s) - yield s + s = (x, y, t) + yield s }.filter { s => !isFinal (s) && valid (s) } - val instances = MazeInstances + val instances = new MazeInstances end Maze @@ -108,8 +106,8 @@ end Maze /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object MazeInstances - extends AgentConstraints[MazeState, MazeObservableState, MazeAction, MazeReward, Randomized]: +class MazeInstances (using probula.RNG) + extends AgentConstraints[MazeState, MazeObservableState, MazeAction, MazeReward, Randomized2]: given enumAction: BoundedEnumerable[MazeAction] = BoundedEnumerableFromList (Up, Down, Left, Right) @@ -119,15 +117,15 @@ object MazeInstances y <- List (1, 2, 3) x <- List (1, 2, 3, 4) result = (x, y, 0) - if Maze.valid (result) + if valid (result) yield (result._1, result._2) BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad - - given schedulerIsFoldable: Foldable[Randomized] = Randomized.randomizedIsFoldable + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad - given canTestInScheduler: CanTestIn[Randomized] = Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized lazy val genMazeState: Gen[MazeState] = for y <- Gen.choose (1, 3) diff --git a/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala b/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala index 2d29fc9e..b86ba548 100644 --- a/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala +++ b/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala @@ -4,10 +4,11 @@ package examples.concrete.windygrid import examples.concrete.windygrid.GridAction.* import cats.{Eq, Foldable, Monad} import cats.kernel.BoundedEnumerable +import cats.syntax.all.* import org.scalacheck.{Arbitrary, Gen} -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 case class GridState (x: Int, y: Int): override def toString: String = s"($x,$y)" @@ -19,8 +20,8 @@ enum GridAction: case R, L, U, D -object WindyGrid extends - Agent[GridState, GridObservableState, GridAction, GridReward, Randomized]: +class WindyGrid (using probula.RNG) extends + Agent[GridState, GridObservableState, GridAction, GridReward, Randomized2]: val WindSpec = Array (0, 0, 0, 1, 1, 1, 2, 2, 1, 0) @@ -54,30 +55,30 @@ object WindyGrid extends val y1 = ((s.y - 1 + WindSpec (s.x - 1)).max (1)).min (7) GridState (x1, y1) - def step (s: GridState) (a: GridAction): Randomized[(GridState, GridReward)] = + def step (s: GridState) (a: GridAction): Randomized2[(GridState, GridReward)] = val s1 = a match case GridAction.R => stepRight (s) case GridAction.L => stepLeft (s) case GridAction.U => stepUp (s) case GridAction.D => stepDown (s) - Randomized.const (s1 -> gridReward (s1) (a)) + Randomized2.const (s1 -> gridReward (s1) (a)) - def initialize: Randomized[GridState] = for - x <- Randomized.repeat (Randomized.between (1, 11)) - y <- Randomized.repeat (Randomized.between (1, 8)) - s = GridState (x, y) if !isFinal (s) - yield s + def initialize: Randomized2[GridState] = { for + x <- Randomized2.between (1, 11) + y <- Randomized2.between (1, 8) + s = GridState (x, y) + yield s }.filter { ! this.isFinal (_) } - val instances = WindyGridInstances + val instances = new WindyGridInstances end WindyGrid /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object WindyGridInstances +class WindyGridInstances (using probula.RNG) extends AgentConstraints[GridState, GridObservableState, GridAction, - GridReward, Randomized]: + GridReward, Randomized2]: given enumAction: BoundedEnumerable[GridAction] = BoundedEnumerableFromList (U, L, R, D) @@ -89,22 +90,24 @@ object WindyGridInstances yield GridState (x, y) BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad - val schedulerIsFoldable: Foldable[Randomized] = Randomized.randomizedIsFoldable - - given canTestInScheduler: CanTestIn[Randomized] = Randomized.canTestInRandomized + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized lazy val genGridState: Gen[GridState] = for x <- Gen.choose (1, 10) y <- Gen.choose (1, 7) yield GridState (x, y) - given arbitraryState: Arbitrary[GridState] = Arbitrary (genGridState) + given arbitraryState: Arbitrary[GridState] = + Arbitrary (genGridState) given eqGridState: Eq[GridState] = Eq.fromUniversalEquals - given arbitraryReward: Arbitrary[GridReward] = Arbitrary (Gen.double) + given arbitraryReward: Arbitrary[GridReward] = + Arbitrary (Gen.double) given rewardArith: Arith[GridReward] = Arith.arithDouble diff --git a/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala b/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala index 903fe117..0166ded3 100644 --- a/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala +++ b/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala @@ -2,6 +2,7 @@ package symsim package laws import scala.language.postfixOps +import cats.syntax.all.* import org.scalacheck.Prop.* import org.scalacheck.Arbitrary @@ -11,7 +12,11 @@ import breeze.stats.distributions.Rand.VariableSeed.* import symsim.concrete.ConcreteExactRL import symsim.concrete.ConcreteQTable -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 + +given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply + /** Laws that have to be obeyed by any refinement of symsim.ConcreetSarsa * @@ -76,7 +81,7 @@ case class ConcreteExpectedSarsaLaws[State, ObservableState, Action] // 0.95 belief that the probability of suboptimal action is ≤ ε. // We check this by computing the posterior and asking CDF (ε) ≥ 0.94 - val successes = trials.take (n).count { _ == true } + val successes = trials.sample (n).count { _ == true } val failures = n - successes // α=1 and β=1 gives a prior, weak flat, unbiased @@ -103,12 +108,12 @@ case class ConcreteExpectedSarsaLaws[State, ObservableState, Action] val os_t = agent.observe (s_t) // call the tested implementation - val sut: Randomized[(Q, State, Action)] = - Randomized.repeat (sarsa.learningEpoch (q_t, s_t, a_t)) + val sut: Randomized2[(Q, State, Action)] = + sarsa.learningEpoch (q_t, s_t, a_t) // call the spec interpreter - val spec: Randomized[(Q, State, Action)] = - Randomized.repeat (bdl.learningEpoch (q_t, s_t, a_t)) + val spec: Randomized2[(Q, State, Action)] = + bdl.learningEpoch (q_t, s_t, a_t) // We do this test by assuming that both distributions are normal // (A generalized test with StudentT would be even better). @@ -124,14 +129,13 @@ case class ConcreteExpectedSarsaLaws[State, ObservableState, Action] // // Extract a univariate distributions over updates - val sutUpdates = sut.map { (q_tt, _, _) => q_tt (os_t, a_t) } - val specUpdates = spec.map { (q_tt, _, _) => q_tt (os_t, a_t) } + val sutUpdates = sut.sample (n).map { (q_tt, _, _) => q_tt (os_t, a_t) } + val specUpdates = spec.sample (n).map { (q_tt, _, _) => q_tt (os_t, a_t) } // A random variable representing differences between updates val diffs = (sutUpdates zip specUpdates) .map { _ - _ } - .take (n) // Infer the posterior on mean update. // Prior: we assume zero reward, with large standard deviation diff --git a/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala b/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala index 3262a5e7..8640fabd 100644 --- a/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala +++ b/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala @@ -4,13 +4,14 @@ package laws import breeze.stats.distributions.{Rand, Beta} import breeze.stats.distributions.Rand.VariableSeed.* import scala.language.postfixOps +import cats.syntax.all.* import org.scalacheck.Prop.* import org.scalacheck.Arbitrary import symsim.concrete.ConcreteExactRL import symsim.concrete.ConcreteQTable -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 /** Laws that have to be obeyed by any refinement of symsim.ConcreetSarsa * @@ -69,7 +70,7 @@ case class ConcreteSarsaLaws[State, ObservableState, Action] // 0.95 belief that the probability of suboptimal action is ≤ ε. // We check this by computing the posterior and asking CDF (ε) ≥ 0.94 - val successes = trials.take (n).count { _ == true } + val successes = trials.sample (n).count { _ == true } val failures = n - successes // α=1 and β=1 gives a prior, weak flat, unbiased @@ -96,12 +97,12 @@ case class ConcreteSarsaLaws[State, ObservableState, Action] val os_t = agent.observe (s_t) // call the tested implementation - val sut: Randomized[(Q, State, Action)] = - Randomized.repeat (sarsa.learningEpoch (q_t, s_t, a_t)) + val sut: Randomized2[(Q, State, Action)] = + sarsa.learningEpoch (q_t, s_t, a_t) // call the spec interpreter - val spec: Randomized[(Q, State, Action)] = - Randomized.repeat (bdl.learningEpoch (q_t, s_t, a_t)) + val spec: Randomized2[(Q, State, Action)] = + bdl.learningEpoch (q_t, s_t, a_t) // We find a Bayesian posterior (analytically) for the mean // difference between them means, and checking whether we @@ -110,14 +111,13 @@ case class ConcreteSarsaLaws[State, ObservableState, Action] // Extract a univariate distributions over updates - val sutUpdates = sut.map { (q_tt, _, _) => q_tt (os_t, a_t) } - val specUpdates = spec.map { (q_tt, _, _) => q_tt (os_t, a_t) } + val sutUpdates = sut.sample (n).map { (q_tt, _, _) => q_tt (os_t, a_t) } + val specUpdates = spec.sample (n).map { (q_tt, _, _) => q_tt (os_t, a_t) } // A random variable representing differences between updates val diffs = (sutUpdates zip specUpdates) .map { _ - _ } - .take (n) // Infer the posterior on mean update. // Prior: we assume zero reward, with large standard deviation @@ -151,9 +151,9 @@ case class ConcreteSarsaLaws[State, ObservableState, Action] "The value of variables in the Q table are not NaN after learning (divergence)" -> forAllNoShrink { (q_t: Q, s_t: State) => - val initials = Randomized.repeat (agent.initialize).take (10) - val schedule = sarsa.learn (q_t, List[Q](), initials) - val q = schedule.head._1 + val initials = agent.initialize.sample (10) + val schedule = sarsa.learn (q_t, List[Q](), initials).sample() + val q = schedule._1 val os_t = agent.observe (s_t) vf.actionValues (q) (os_t) .values diff --git a/src/test/scala/symsim/ExperimentSpec.scala b/src/test/scala/symsim/ExperimentSpec.scala index 91eeb29a..7219fc38 100644 --- a/src/test/scala/symsim/ExperimentSpec.scala +++ b/src/test/scala/symsim/ExperimentSpec.scala @@ -5,8 +5,13 @@ import java.io.PrintWriter import java.nio.charset.StandardCharsets import symsim.concrete.ConcreteQTable -import symsim.concrete.Randomized -import symsim.concrete.Randomized.sample +import symsim.concrete.Randomized2 + +import cats.syntax.all.* +import symsim.concrete.Randomized2.* + +given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply trait ExperimentSpec[State, ObservableState, Action] extends org.scalatest.freespec.AnyFreeSpec, @@ -63,14 +68,15 @@ trait ExperimentSpec[State, ObservableState, Action] def eval ( setup: concrete.ConcreteExactRL [State, ObservableState, Action], policies: List[setup.Policy], - initials: Option[Randomized[State]] = None, + initials: Option[Randomized2[State]] = None, noOfEpisodes: Int = 5 ): EvaluationResults = - val ss: Randomized[State] = initials.getOrElse (setup.agent.initialize) + val ss: Randomized2[State] = initials.getOrElse (setup.agent.initialize) for p <- policies - episodeRewards: Randomized[Randomized[Double]] = setup.evaluate (p, ss) - rewards: Randomized[Double] = episodeRewards.map { e => e.sample () } - yield rewards.take (noOfEpisodes).toList + episodeRewards: Randomized2[Randomized2[Double]] = + setup.evaluate (p, ss) + rewards: Randomized2[Double] = episodeRewards.map { e => e.sample () } + yield rewards.sample (noOfEpisodes).toList def mean(l: List[Double]): Double = diff --git a/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala b/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala index 88a11f34..8b04a5b6 100644 --- a/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala +++ b/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala @@ -3,7 +3,10 @@ package concrete import symsim.examples.concrete.mountaincar.MountainCar -import MountainCar.instances.given +private val mountainCar = + new MountainCar (using spire.random.rng.SecureJava.apply) + +import mountainCar.instances.given /** This test is just a sanity check - it mostly tests Bdl against * itself (so an equality check). @@ -12,7 +15,7 @@ class BdlConcreteExpectedSarsaIsExpectedSarsaSpec extends SymSimSpec: val csarsa = BdlConcreteExpectedSarsa ( - agent = MountainCar, + agent = mountainCar, alpha = 0.1, gamma = 0.2, epsilon0 = 0.0, // The update distribution test requires low ε for stability diff --git a/src/test/scala/symsim/concrete/BdlConcreteSarsaIsSarsa.scala b/src/test/scala/symsim/concrete/BdlConcreteSarsaIsSarsa.scala index 8f8c6870..a5f2d238 100644 --- a/src/test/scala/symsim/concrete/BdlConcreteSarsaIsSarsa.scala +++ b/src/test/scala/symsim/concrete/BdlConcreteSarsaIsSarsa.scala @@ -3,7 +3,11 @@ package concrete import symsim.examples.concrete.mountaincar.MountainCar -import MountainCar.instances.given +private val mountainCar = + new MountainCar (using spire.random.rng.SecureJava.apply) + +import mountainCar.instances.enumState +import mountainCar.instances.enumAction /** This test is just a sanity check - it mostly tests Bdl against * itself (so an equality check). @@ -12,7 +16,7 @@ class BdlConcreteSarsaIsSarsaSpec extends SymSimSpec: val csarsa = BdlConcreteSarsa ( - agent = MountainCar, + agent = mountainCar, alpha = 0.1, gamma = 0.2, epsilon0 = 0.0, // the update distribution test requires low ε for stability diff --git a/src/test/scala/symsim/concrete/ConcreteExpectedSarsaIsExpectedSarsa.scala b/src/test/scala/symsim/concrete/ConcreteExpectedSarsaIsExpectedSarsa.scala index c4ac830b..f432f7f3 100644 --- a/src/test/scala/symsim/concrete/ConcreteExpectedSarsaIsExpectedSarsa.scala +++ b/src/test/scala/symsim/concrete/ConcreteExpectedSarsaIsExpectedSarsa.scala @@ -3,13 +3,11 @@ package concrete import symsim.examples.concrete.mountaincar.MountainCar -import MountainCar.instances.given - class ConcreteExpectedSarsaIsSarsaSpec extends SymSimSpec: val csarsa = ConcreteExpectedSarsa ( - agent = MountainCar, + agent = new MountainCar, alpha = 0.1, gamma = 0.2, epsilon0 = 0.003, // The update distribution test requires low ε for stability diff --git a/src/test/scala/symsim/concrete/ConcreteQLearningIsSarsaSpec.scala b/src/test/scala/symsim/concrete/ConcreteQLearningIsSarsaSpec.scala index 740db8df..94b3fd48 100644 --- a/src/test/scala/symsim/concrete/ConcreteQLearningIsSarsaSpec.scala +++ b/src/test/scala/symsim/concrete/ConcreteQLearningIsSarsaSpec.scala @@ -3,7 +3,9 @@ package concrete import symsim.examples.concrete.mountaincar.MountainCar -import MountainCar.instances.given +private val mountainCar = + new MountainCar (using spire.random.rng.SecureJava.apply) +import mountainCar.instances.{enumAction, enumState} /** The name of this test might be amusing, but since the test * only exercise single epoch properties, a QLearning @@ -14,7 +16,7 @@ class ConcreteQLearningIsSarsaSpec extends SymSimSpec: val qLearning = ConcreteQLearning ( - agent = MountainCar, + agent = mountainCar, alpha = 0.1, gamma = 0.2, epsilon0 = 0.0, // The update distribution test requires low ε for stability diff --git a/src/test/scala/symsim/concrete/ConcreteSarsaIsSarsaSpec.scala b/src/test/scala/symsim/concrete/ConcreteSarsaIsSarsaSpec.scala index ce5d5868..3a7ca714 100644 --- a/src/test/scala/symsim/concrete/ConcreteSarsaIsSarsaSpec.scala +++ b/src/test/scala/symsim/concrete/ConcreteSarsaIsSarsaSpec.scala @@ -3,13 +3,15 @@ package concrete import symsim.examples.concrete.mountaincar.MountainCar -import MountainCar.instances.given +private val mountainCar = + new MountainCar (using spire.random.rng.SecureJava.apply) +import mountainCar.instances.{enumAction, enumState} class ConcreteSarsaIsSarsaSpec extends SymSimSpec: val csarsa = ConcreteSarsa ( - agent = MountainCar, + agent = mountainCar, alpha = 0.1, gamma = 0.2, epsilon0 = 0.003, // The update distribution test requires low ε for stability diff --git a/src/test/scala/symsim/concrete/ConcreteSarsaSpec.scala b/src/test/scala/symsim/concrete/ConcreteSarsaSpec.scala index 31d4e990..d5f2ad68 100644 --- a/src/test/scala/symsim/concrete/ConcreteSarsaSpec.scala +++ b/src/test/scala/symsim/concrete/ConcreteSarsaSpec.scala @@ -1,17 +1,18 @@ package symsim package concrete +private val unitAgent = + new UnitAgent (using spire.random.rng.SecureJava.apply) +import unitAgent.instances.enumAction + class ConcreteSarsaSpec extends org.scalatest.freespec.AnyFreeSpec, org.scalatest.matchers.should.Matchers: - // Import evidence that states and actions can be enumerated - import UnitAgent.* - val C = 555555 val sarsa = ConcreteSarsa ( - agent = UnitAgent, + agent = unitAgent, alpha = 0.1, gamma = 0.1, epsilon0 = 0.2, // explore vs exploit ratio @@ -25,11 +26,10 @@ class ConcreteSarsaSpec } "learnN shouldn't overflow stack (learnN is tailrec, each episode tailrec)" in { - val initials: Randomized[UnitState] = - Randomized.repeat (UnitAgent.initialize) - val result: Randomized[(sarsa.vf.Q, List[sarsa.vf.Q])] = - sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q](), initials.take (C)) - try result.size == 1 + val initials: Randomized2[UnitState] = unitAgent.initialize + val result: Randomized2[(sarsa.vf.Q, List[sarsa.vf.Q])] = + sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q](), initials.sample (C)) + try result.sample() catch case e => fail (s"Forcing result of learning overflows (${e.toString})") } @@ -44,7 +44,7 @@ class ConcreteSarsaSpec // also with the immediate final state 'learn' is not really tested here "learn is tail recursive, no stack overflow (regression)" in { val result = sarsa.learningEpisode ((sarsa.vf.initialize, List[sarsa.vf.Q](), sarsa.ε0), ()) - result.head + result.sample() } "runQ is tail recursive, no stack overflow (regression)" in { diff --git a/src/test/scala/symsim/concrete/RandomizedSpec.scala b/src/test/scala/symsim/concrete/RandomizedSpec.scala index 398f494f..31cd5bf6 100644 --- a/src/test/scala/symsim/concrete/RandomizedSpec.scala +++ b/src/test/scala/symsim/concrete/RandomizedSpec.scala @@ -130,14 +130,4 @@ class RandomizedSpec } } - "Randomized Foldable Instance" - { - - "check that foldLeft works (sum)" in { - val r = Randomized.repeat (Randomized.oneOf (1)).take (C) - val sum = Randomized.randomizedIsFoldable - .foldLeft[Int, Int] (r, 0) { _ + _ } - assert (sum == C) - } - } - end RandomizedSpec diff --git a/src/test/scala/symsim/concrete/UnitAgent.scala b/src/test/scala/symsim/concrete/UnitAgent.scala index 6fd73029..747c0005 100644 --- a/src/test/scala/symsim/concrete/UnitAgent.scala +++ b/src/test/scala/symsim/concrete/UnitAgent.scala @@ -10,28 +10,27 @@ type UnitState = Unit type UnitAction = Unit type UnitReward = Double -object UnitAgent - extends Agent[UnitState, UnitState, UnitAction, UnitReward, Randomized]: +class UnitAgent (using probula.RNG) + extends Agent[UnitState, UnitState, UnitAction, UnitReward, Randomized2]: override def isFinal (s: UnitState): Boolean = true override def observe (s: UnitState): UnitState = s - override def step (s: UnitState) (a: UnitAction): Randomized[(UnitState, UnitReward)] = - Randomized.const (() -> 0.1) - override def initialize: Randomized[UnitState] = Randomized.const (()) - override val instances = UnitInstances + override def step (s: UnitState) (a: UnitAction): Randomized2[(UnitState, UnitReward)] = + Randomized2.const (() -> 0.1) + override def initialize: Randomized2[UnitState] = Randomized2.const (()) + override val instances = new UnitInstances /** Here is a proof that our types actually deliver on everything that an Agent * needs to be able to do to work in the framework. */ -object UnitInstances - extends AgentConstraints[UnitState, UnitState, UnitAction, UnitReward, Randomized]: - given enumAction: BoundedEnumerable[UnitAction] = BoundedEnumerableFromList (()) - given enumState: BoundedEnumerable[UnitState] = BoundedEnumerableFromList (()) - given schedulerIsMonad: Monad[Randomized] = Randomized.randomizedIsMonad - given schedulerIsFoldable: Foldable[Randomized] = Randomized.randomizedIsFoldable - given canTestInScheduler: CanTestIn[Randomized] = Randomized.canTestInRandomized - lazy val genUnitState: Gen[UnitState] = Gen.const (()) - given arbitraryState: Arbitrary[UnitState] = Arbitrary (genUnitState) - given eqUnitState: Eq[UnitState] = Eq.fromUniversalEquals - given arbitraryReward: Arbitrary[UnitReward] = Arbitrary (Gen.double) - given rewardArith: Arith[UnitReward] = Arith.arithDouble +class UnitInstances (using probula.RNG) + extends AgentConstraints[UnitState, UnitState, UnitAction, UnitReward, Randomized2]: + given enumAction: BoundedEnumerable[UnitAction] = BoundedEnumerableFromList (()) + given enumState: BoundedEnumerable[UnitState] = enumAction + given schedulerIsMonad: Monad[Randomized2] = Randomized2.randomizedIsMonad + given canTestInScheduler: CanTestIn[Randomized2] = Randomized2.canTestInRandomized + lazy val genUnitState: Gen[UnitState] = Gen.const (()) + given arbitraryState: Arbitrary[UnitState] = Arbitrary (genUnitState) + given eqUnitState: Eq[UnitState] = Eq.fromUniversalEquals + given arbitraryReward: Arbitrary[UnitReward] = Arbitrary (Gen.double) + given rewardArith: Arith[UnitReward] = Arith.arithDouble diff --git a/src/test/scala/symsim/concrete/UnitAgentExperimentsSpec.scala b/src/test/scala/symsim/concrete/UnitAgentExperimentsSpec.scala index 956e769b..e2bd1165 100644 --- a/src/test/scala/symsim/concrete/UnitAgentExperimentsSpec.scala +++ b/src/test/scala/symsim/concrete/UnitAgentExperimentsSpec.scala @@ -1,14 +1,15 @@ package symsim package concrete +private val unitAgent = + new UnitAgent (using spire.random.rng.SecureJava.apply) +import unitAgent.instances.enumAction + class UnitAgentExperiments extends ExperimentSpec[UnitState, UnitState, UnitAction]: - // Import evidence that states and actions can be enumerated - import UnitAgent.* - val sarsa = symsim.concrete.ConcreteSarsa ( - agent = UnitAgent, + agent = unitAgent, alpha = 0.1, gamma = 0.1, epsilon0 = 0.05, // explore vs exploit ratio @@ -21,7 +22,7 @@ class UnitAgentExperiments } val qLearning = symsim.concrete.ConcreteQLearning ( - agent = UnitAgent, + agent = unitAgent, alpha = 0.1, gamma = 0.1, epsilon0 = 0.05, // explore vs exploit ratio diff --git a/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala index 9cdd6130..1cb2e884 100644 --- a/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala @@ -7,5 +7,5 @@ import laws.EpisodicLaws class CarIsAgentSpec extends SymSimSpec: - checkAll ("concrete.braking.Car is an Agent", AgentLaws (Car).laws) - checkAll ("concrete.braking.Car is Episodic", EpisodicLaws (Car).laws) + checkAll ("concrete.braking.Car is an Agent", AgentLaws (new Car).laws) + checkAll ("concrete.braking.Car is Episodic", EpisodicLaws (new Car).laws) diff --git a/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala b/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala index 427d153c..e11e2f61 100644 --- a/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala +++ b/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala @@ -1,58 +1,62 @@ package symsim package examples.concrete.braking +import cats.syntax.all.* + import org.scalacheck.Gen import org.scalacheck.Prop.* import symsim.CanTestIn.given import symsim.concrete.Randomized.given -import Car.instances.{arbitraryState, arbitraryAction} +import car.instances.{arbitraryState, arbitraryAction} // To eliminate the warning on CarSpec, until scalacheck makes it open import scala.language.adhocExtensions +private val car = new Car (using spire.random.rng.SecureJava.apply) + /** Sanity tests for symsim.concrete.braking */ object CarSpec extends org.scalacheck.Properties ("braking.Car"): - property("A stopped car cannot move, however much you break") = + property ("A stopped car cannot move, however much you break") = forAll { (s: CarState, a: CarAction) => - for (s1, r) <- Car.step (s.copy (v = 0.0)) (a) + for (s1, r) <- car.step (s.copy (v = 0.0)) (a) yield s1.p == s.p } - property("The car cannot move backwards by braking") = + property ("The car cannot move backwards by braking") = forAll { (s: CarState, a: CarAction) => - for (s1, r) <- Car.step (s) (a) + for (s1, r) <- car.step (s) (a) yield (s.v != 0 ==> s1.p >= s.p) } - property("Position never becomes negative") = + property ("Position never becomes negative") = forAll { (s: CarState, a: CarAction) => - for (s1, r) <- Car.step (s) (a) + for (s1, r) <- car.step (s) (a) yield s1.p >= 0.0 } - property("Reward is valid 1") = + property ("Reward is valid 1") = forAll { (s1: CarState, s2: CarState, a: CarAction) => for - (_, r1) <- Car.step (CarState (v = s1.v, p = s1.p min s2.p)) (a) - (_, r2) <- Car.step (CarState (v = s1.v, p = s1.p max s2.p)) (a) + (_, r1) <- car.step (CarState (v = s1.v, p = s1.p min s2.p)) (a) + (_, r2) <- car.step (CarState (v = s1.v, p = s1.p max s2.p)) (a) yield r1 >= r2 } - property("Reward is valid 2") = + property ("Reward is valid 2") = forAll { (s1: CarState, s2: CarState, a: CarAction) => for - (_, r1) <- Car.step (CarState (v = s1.v min s2.v, p = s1.p)) (a) - (_, r2) <- Car.step (CarState (v = s1.v max s2.v, p = s1.p)) (a) + (_, r1) <- car.step (CarState (v = s1.v min s2.v, p = s1.p)) (a) + (_, r2) <- car.step (CarState (v = s1.v max s2.v, p = s1.p)) (a) yield r1 >= r2 } - property("Velocity is random") = - val states = Car.initialize.take(5).toList + property ("Velocity is random") = + val states = car.initialize.sample (5) val vs = states.map(s => s.v) - vs.combinations(2).exists(pair => pair(0) != pair(1)) + vs.combinations (2).exists { pair => pair (0) != pair (1) } diff --git a/src/test/scala/symsim/examples/concrete/braking/Experiments.scala b/src/test/scala/symsim/examples/concrete/braking/Experiments.scala index 089eb90a..e3d0a42d 100644 --- a/src/test/scala/symsim/examples/concrete/braking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/braking/Experiments.scala @@ -1,17 +1,16 @@ package symsim package examples.concrete.braking -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 + +private val car = new Car (using spire.random.rng.SecureJava.apply) +import car.instances.{enumAction, enumState} class Experiments extends ExperimentSpec[CarState, CarObservableState, CarAction]: - // Import evidence that states and actions can be enumerated - import Car.* - import Car.instances.given - val sarsa = symsim.concrete.ConcreteSarsa ( - agent = Car, + agent = car, alpha = 0.1, gamma = 0.1, epsilon0 = 0.05, @@ -25,7 +24,7 @@ class Experiments .flatMap { _.headOption } .toList // FIXME: I tentatively just try from an adhoc state - val evalInitial = Randomized.const (CarState (10.0, 0.0)) + val evalInitial = Randomized2.const (CarState (10.0, 0.0)) eval (sarsa, policies, Some (evalInitial)) .save ("braking.csv") } diff --git a/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala b/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala index b3020a78..374e64ae 100644 --- a/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala @@ -2,13 +2,15 @@ package symsim package examples.concrete.cartpole // Import evidence that states and actions can be enumerated -import CartPole.instances.given +private val cartPole = + new CartPole (using spire.random.rng.SecureJava.apply) +import cartPole.instances.{enumAction, enumState} class Experiments extends ExperimentSpec[CartPoleState, CartPoleObservableState, CartPoleAction]: val sarsa = symsim.concrete.ConcreteSarsa ( - agent = CartPole, + agent = cartPole, alpha = 0.1, gamma = 0.1, epsilon0 = 0.05, diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingIsAgentSpec.scala index 354c580d..f187cc4a 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingIsAgentSpec.scala @@ -4,8 +4,11 @@ package examples.concrete.cliffWalking import laws.AgentLaws import laws.EpisodicLaws +private val cliffWalking = + new CliffWalking (using spire.random.rng.SecureJava.apply) + class CliffWalkingIsAgentSpec extends SymSimSpec: - checkAll ("concrete.cliffWalking.CliffWalking is an Agent", AgentLaws (CliffWalking).laws) - checkAll ("concrete.cliffWalking.CliffWalking is Episodic", EpisodicLaws (CliffWalking).laws) + checkAll ("concrete.cliffWalking.CliffWalking is an Agent", AgentLaws (cliffWalking).laws) + checkAll ("concrete.cliffWalking.CliffWalking is Episodic", EpisodicLaws (cliffWalking).laws) diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala index c0b7be86..12a19562 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala @@ -1,12 +1,15 @@ package symsim.examples.concrete.cliffWalking +import cats.syntax.all.* import org.scalacheck.Gen import org.scalacheck.Prop.forAll import symsim.CanTestIn.given import symsim.concrete.Randomized.given -import CliffWalking.instances.{arbitraryState, arbitraryAction} +private val cliffWalking = + new CliffWalking (using spire.random.rng.SecureJava.apply) +import cliffWalking.instances.{arbitraryState, arbitraryAction, canTestInScheduler} // Eliminate the warning on CliffWalkingSpec, until scalacheck makes it open import scala.language.adhocExtensions @@ -16,6 +19,6 @@ object CliffWalkingSpec property("All rewards are negative") = forAll { (s: CWState, a: CWAction) => - for (_, r) <- CliffWalking.step (s) (a) + for (_, r) <- cliffWalking.step (s) (a) yield r < 0 } diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala index 7d04e639..ddbc62ba 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala @@ -1,16 +1,18 @@ package symsim package examples.concrete.cliffWalking -import CliffWalking.instances.given +private val cliffWalking = + new CliffWalking (using spire.random.rng.SecureJava.apply) +import cliffWalking.instances.{enumAction, enumState} class Experiments extends ExperimentSpec[CWState, CWObservableState, CWAction]: // Import evidence that states and actions can be enumerated - import CliffWalking.* + import cliffWalking.* val sarsa = symsim.concrete.ConcreteSarsa ( - agent = CliffWalking, + agent = cliffWalking, alpha = 0.1, gamma = 0.1, epsilon0 = 0.1, diff --git a/src/test/scala/symsim/examples/concrete/golf/Experiments.scala b/src/test/scala/symsim/examples/concrete/golf/Experiments.scala index ba253033..63232c42 100644 --- a/src/test/scala/symsim/examples/concrete/golf/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/golf/Experiments.scala @@ -1,14 +1,17 @@ package symsim package examples.concrete.golf +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val golf: Golf = new Golf + +import golf.instances.{enumAction, enumState} + class Experiments extends ExperimentSpec[GolfState, GolfState, GolfAction]: - import Golf.* - import Golf.instances.given - val sarsa = symsim.concrete.ConcreteSarsa ( - agent = Golf, + agent = golf, alpha = 0.1, gamma = 0.1, epsilon0 = 0.1, diff --git a/src/test/scala/symsim/examples/concrete/golf/GolfIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/golf/GolfIsAgentSpec.scala index 350e7402..4c6251fc 100644 --- a/src/test/scala/symsim/examples/concrete/golf/GolfIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/golf/GolfIsAgentSpec.scala @@ -4,8 +4,10 @@ package examples.concrete.golf import laws.AgentLaws import laws.EpisodicLaws +private val golf = new Golf (using spire.random.rng.SecureJava.apply) + class GolfIsAgentSpec extends SymSimSpec: - checkAll ("concrete.golf.Golf is an Agent", AgentLaws (Golf).laws) - checkAll ("concrete.golf.Golf is Episodic", EpisodicLaws (Golf).laws) + checkAll ("concrete.golf.Golf is an Agent", AgentLaws (golf).laws) + checkAll ("concrete.golf.Golf is Episodic", EpisodicLaws (golf).laws) diff --git a/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala b/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala index d18b9e60..07cc0570 100644 --- a/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala +++ b/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala @@ -1,36 +1,41 @@ package symsim.examples.concrete.golf +import cats.syntax.all.* + import org.scalacheck.Gen import org.scalacheck.Prop.* import symsim.CanTestIn.given import symsim.concrete.ConcreteSarsa import symsim.concrete.Randomized -import symsim.concrete.Randomized.given // Eliminate the warning on GolfSpec until scalacheck marks Properties it open import scala.language.adhocExtensions +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val golf = new Golf + class GolfSpec extends org.scalacheck.Properties ("Golf"): - import Golf.instances.{arbitraryState, arbitraryAction, enumState, enumAction} + import golf.instances.{enumAction, enumState, arbitraryState, arbitraryAction} // The number of episodes set to zero, as it is ignored in tests below - val sarsa = ConcreteSarsa (Golf, 0.1, 0.1, 0.1, 0) + val sarsa = ConcreteSarsa (golf, 0.1, 0.1, 0.1, 0) import sarsa.vf.apply property ("There is no action that leads agent to initial state") = forAll { (s: GolfState, a: GolfAction) => - for (s1, r) <- Golf.step (s) (a) - yield s1 != Golf.StartState + for (s1, r) <- golf.step (s) (a) + yield s1 != golf.StartState } property ("Ball stuck in the sand until using club D") = forAll { (a: GolfAction) => - for (s1, r) <- Golf.step (Golf.SandState) (a) - yield a._1 != Club.D ==> (s1 == Golf.SandState) + for (s1, r) <- golf.step (golf.SandState) (a) + yield a._1 != Club.D ==> (s1 == golf.SandState) } property ("Q-table values are non-positive") = @@ -41,6 +46,7 @@ class GolfSpec } property ("Using club D in the sand is the best trained action after 100 episodes") = - val initials = Randomized.repeat(Randomized.const(Golf.StartState)).take(200) + val initials = Randomized.repeat(Randomized.const(golf.StartState)).take(200) val q = sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q](), initials) - q.take(50).forall { (q, _) => sarsa.vf.bestAction (q) (Golf.SandState)._1 == Club.D } + q.sample (50) + .forall { (q, _) => sarsa.vf.bestAction (q) (golf.SandState)._1 == Club.D } diff --git a/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala b/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala index ac21db5d..99ba079e 100644 --- a/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala @@ -1,15 +1,19 @@ package symsim package examples.concrete.mountaincar +private val mountainCar = + new MountainCar (using spire.random.rng.SecureJava.apply) +import mountainCar.instances.{enumAction, enumState} + class Experiments extends ExperimentSpec[CarState, CarObservableState, CarAction]: // Import evidence that states and actions can be enumerated - import MountainCar.* - import MountainCar.instances.given + import mountainCar.* + import mountainCar.instances.given val sarsa = symsim.concrete.ConcreteSarsa ( - agent = MountainCar, + agent = mountainCar, alpha = 0.1, gamma = 0.1, epsilon0 = 0.05, diff --git a/src/test/scala/symsim/examples/concrete/mountaincar/MountainCarIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/mountaincar/MountainCarIsAgentSpec.scala index 46ded914..2d50f735 100644 --- a/src/test/scala/symsim/examples/concrete/mountaincar/MountainCarIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/mountaincar/MountainCarIsAgentSpec.scala @@ -4,10 +4,13 @@ package examples.concrete.mountaincar import laws.AgentLaws import laws.EpisodicLaws +private val mountainCar = + new MountainCar (using spire.random.rng.SecureJava.apply) + class MountainCarIsAgentSpec extends SymSimSpec: - checkAll ("concrete.mountaincar.MountainCar is an Agent", AgentLaws (MountainCar).laws) + checkAll ("concrete.mountaincar.MountainCar is an Agent", AgentLaws (mountainCar).laws) // Randomized testing of this law is too simplistic for mountaincar to // reliably pass the test, so it is deactivated for now. A symbolic test' // could be more efficient. diff --git a/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala b/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala index 1226e195..4f12e6d8 100644 --- a/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala @@ -1,15 +1,17 @@ package symsim package examples.concrete.pumping +private val pump = new Pump (using spire.random.rng.SecureJava.apply) + class Experiments - extends ExperimentSpec[PumpState,ObservablePumpState,PumpAction]: + extends ExperimentSpec[PumpState, ObservablePumpState, PumpAction]: // Import evidence that states and actions can be enumerated - import Pump.* - import Pump.instances.given + import pump.* + import pump.instances.given val sarsa = symsim.concrete.ConcreteSarsa ( - agent = Pump, + agent = pump, alpha = 0.1, gamma = 0.9, epsilon0 = 0.05, diff --git a/src/test/scala/symsim/examples/concrete/pumping/PumpIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/pumping/PumpIsAgentSpec.scala index ed68f84c..5c59afe0 100644 --- a/src/test/scala/symsim/examples/concrete/pumping/PumpIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/pumping/PumpIsAgentSpec.scala @@ -4,8 +4,10 @@ package examples.concrete.pumping import laws.AgentLaws import laws.EpisodicLaws +private val pump = new Pump (using spire.random.rng.SecureJava.apply) + class PumpIsAgentSpec extends SymSimSpec: - checkAll ("concrete.pumping.Pump is an Agent", AgentLaws (Pump).laws) - checkAll ("concrete.pumping.Pump is Episodic", EpisodicLaws (Pump).laws) + checkAll ("concrete.pumping.Pump is an Agent", AgentLaws (pump).laws) + checkAll ("concrete.pumping.Pump is Episodic", EpisodicLaws (pump).laws) diff --git a/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectConst.scala b/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectConst.scala index 945510de..3a25487b 100644 --- a/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectConst.scala +++ b/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectConst.scala @@ -1,15 +1,16 @@ package symsim package examples.concrete.simplebandit -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 -val BanditObjConst = new Bandit(List(Randomized.const(-0.2), - Randomized.const(0.2), - Randomized.const(0.4), - Randomized.const(-0.1), - Randomized.const(0.3), - Randomized.const(0.8), - Randomized.const(0.6), - Randomized.const(-0.6), - Randomized.const(0.0), - Randomized.const(0.7))) +val BanditObjConst = new Bandit (List ( + Randomized2.const (-0.2), + Randomized2.const ( 0.2), + Randomized2.const ( 0.4), + Randomized2.const (-0.1), + Randomized2.const ( 0.3), + Randomized2.const ( 0.8), + Randomized2.const ( 0.6), + Randomized2.const (-0.6), + Randomized2.const ( 0.0), + Randomized2.const ( 0.7))) diff --git a/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala b/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala index d6049e4f..c0be5511 100644 --- a/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala +++ b/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala @@ -1,15 +1,16 @@ package symsim package examples.concrete.simplebandit -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 -val BanditObjGaussian = new Bandit(List(Randomized.gaussian(-0.2, 1), - Randomized.gaussian(0.2, 1), - Randomized.gaussian(0.4, 1), - Randomized.gaussian(-0.1, 1), - Randomized.gaussian(0.3, 1), - Randomized.gaussian(0.8, 1), - Randomized.gaussian(0.6, 1), - Randomized.gaussian(-0.6, 1), - Randomized.gaussian(0.0, 1), - Randomized.gaussian(0.7, 1))) +val BanditObjGaussian = new Bandit (List ( + Randomized2.gaussian (-0.2, 1), + Randomized2.gaussian ( 0.2, 1), + Randomized2.gaussian ( 0.4, 1), + Randomized2.gaussian (-0.1, 1), + Randomized2.gaussian ( 0.3, 1), + Randomized2.gaussian ( 0.8, 1), + Randomized2.gaussian ( 0.6, 1), + Randomized2.gaussian (-0.6, 1), + Randomized2.gaussian ( 0.0, 1), + Randomized2.gaussian ( 0.7, 1))) diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/BdlExperiments.scala b/src/test/scala/symsim/examples/concrete/simplemaze/BdlExperiments.scala index a6ba35c2..5b0e29c8 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/BdlExperiments.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/BdlExperiments.scala @@ -1,13 +1,16 @@ package symsim package examples.concrete.simplemaze -import Maze.instances.given +private val maze = new Maze (using spire.random.rng.SecureJava.apply) + +import maze.instances.{enumAction, enumState} +import MazeAction.* class BdlExperiments extends ExperimentSpec[MazeState,MazeObservableState,MazeAction]: val sarsa = symsim.concrete.BdlConcreteSarsa ( - agent = Maze, + agent = maze, alpha = 0.2, gamma = 1, epsilon0 = 0.05, diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/ExpectedSarsaExperiments.scala b/src/test/scala/symsim/examples/concrete/simplemaze/ExpectedSarsaExperiments.scala index a03fe0a9..60be3490 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/ExpectedSarsaExperiments.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/ExpectedSarsaExperiments.scala @@ -1,13 +1,16 @@ package symsim package examples.concrete.simplemaze -import Maze.instances.given +private val maze = new Maze (using spire.random.rng.SecureJava.apply) +import maze.instances.{enumAction, enumState} + +import MazeAction.* class ExpectedSarsaExperiments extends ExperimentSpec[MazeState,MazeObservableState,MazeAction]: val sarsa = symsim.concrete.ConcreteExpectedSarsa ( - agent = Maze, + agent = maze, alpha = 0.1, gamma = 1, epsilon0 = 0.05, diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/MazeIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/simplemaze/MazeIsAgentSpec.scala index e4fa5286..1d3ab117 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/MazeIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/MazeIsAgentSpec.scala @@ -4,8 +4,10 @@ package examples.concrete.simplemaze import laws.AgentLaws import laws.EpisodicLaws +private val maze = new Maze (using spire.random.rng.SecureJava.apply) + class MazeIsAgentSpec extends SymSimSpec: - checkAll ("concrete.simplemaze.Maze is an Agent", AgentLaws (Maze).laws) - checkAll ("concrete.simplemaze.Maze is Episodic", EpisodicLaws (Maze).laws) + checkAll ("concrete.simplemaze.Maze is an Agent", AgentLaws (maze).laws) + checkAll ("concrete.simplemaze.Maze is Episodic", EpisodicLaws (maze).laws) diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala b/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala index cad5b8aa..4d011ad2 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala @@ -1,12 +1,16 @@ package symsim.examples.concrete.simplemaze +import cats.syntax.all.* + import org.scalacheck.Gen import org.scalacheck.Prop.forAll import symsim.CanTestIn.given import symsim.concrete.Randomized.given -import Maze.instances.{arbitraryState, arbitraryAction} +private val maze = new Maze (using spire.random.rng.SecureJava.apply) +import maze.instances.{arbitraryAction, arbitraryState} +import MazeAction.* // Eliminate the warning on MazeSpec, until scalacheck makes Properties open import scala.language.adhocExtensions @@ -16,30 +20,30 @@ object MazeSpec property ("Agent near to left wall, cannot go left") = forAll { (s: MazeState) => - for (s1, r) <- Maze.step (1, s._2, s._3) (Left) + for (s1, r) <- maze.step (1, s._2, s._3) (Left) yield s1._1 == 1 } property ("Agent near to right wall, cannot go right") = forAll { (s: MazeState) => - for (s1, r) <- Maze.step (4, s._2, s._3) (Right) + for (s1, r) <- maze.step (4, s._2, s._3) (Right) yield s1._1 == 4 } property ("Agent near to up wall, cannot go up") = forAll { (s: MazeState) => - for (s1, r) <- Maze.step (s._1, 3, s._3) (Up) + for (s1, r) <- maze.step (s._1, 3, s._3) (Up) yield s1._2 == 3 } property ("Agent near to down wall, cannot go down") = forAll { (s: MazeState) => - for (s1, r) <- Maze.step (s._1, 1, s._3) (Down) + for (s1, r) <- maze.step (s._1, 1, s._3) (Down) yield s1._2 == 1 } property ("Agent cannot go into block") = forAll { (s: MazeState, a: MazeAction) => - for (s1, r) <- Maze.step (s) (a) + for (s1, r) <- maze.step (s) (a) yield (s1._1, s._2) != (2, 2) } diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala b/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala index 66c2296f..c614b482 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala @@ -1,17 +1,24 @@ package symsim package examples.concrete.simplemaze -import Maze.instances.given +given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply + +private val maze = new Maze + +import MazeAction.* class SarsaExperiments extends ExperimentSpec[MazeState, MazeObservableState, MazeAction]: + import maze.instances.{enumAction, enumState} + val sarsa = symsim.concrete.ConcreteSarsa ( - agent = Maze, + agent = maze, alpha = 0.1, gamma = 1, epsilon0 = 0.1, - episodes = 60000, + episodes = 1000000, ) s"SimpleMaze experiment with ${sarsa}" in { diff --git a/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala b/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala index b0ca480e..a2071218 100644 --- a/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala @@ -1,15 +1,17 @@ package symsim package examples.concrete.windygrid +private val windyGrid = new WindyGrid (using spire.random.rng.SecureJava.apply) + class Experiments extends ExperimentSpec[GridState, GridObservableState, GridAction]: // Import evidence that states and actions can be enumerated - import WindyGrid.* - import WindyGrid.instances.given + import windyGrid.* + import windyGrid.instances.given val sarsa = symsim.concrete.ConcreteSarsa ( - agent = WindyGrid, + agent = windyGrid, alpha = 0.1, gamma = 0.1, epsilon0 = 0.1, diff --git a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridIsAgentSpec.scala index 0458cd94..ad7d120d 100644 --- a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridIsAgentSpec.scala @@ -3,7 +3,9 @@ package examples.concrete.windygrid import laws.AgentLaws +private val windyGrid = new WindyGrid (using spire.random.rng.SecureJava.apply) + class WindyGridIsAgentSpec extends SymSimSpec: - checkAll ("concrete.windygrid.WindyGrid is an Agent", AgentLaws (WindyGrid).laws) + checkAll ("concrete.windygrid.WindyGrid is an Agent", AgentLaws (windyGrid).laws) diff --git a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala index 6533df25..c152d988 100644 --- a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala +++ b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala @@ -1,14 +1,18 @@ package symsim.examples.concrete.windygrid +import cats.syntax.all.* + import org.scalacheck.Gen import org.scalacheck.Prop.* import symsim.CanTestIn.given import symsim.concrete.ConcreteSarsa -import symsim.concrete.Randomized -import symsim.concrete.Randomized.given +import symsim.concrete.Randomized2 +import symsim.concrete.Randomized2.given + +given spire.random.rng.SecureJava = spire.random.rng.SecureJava.apply -import WindyGrid.instances.{arbitraryAction, arbitraryState, enumState, enumAction} +private val windyGrid = new WindyGrid // Eliminate the warning on WindyGridSpec until scalacheck makes Properties open import scala.language.adhocExtensions @@ -16,32 +20,35 @@ import scala.language.adhocExtensions object WindyGridSpec extends org.scalacheck.Properties ("WindyGrid"): + import windyGrid.instances.{arbitraryAction, arbitraryState, enumState, enumAction} + property ("Up and Down will never affect the x value") = forAll { (s: GridState) => for - (s1, r) <- WindyGrid.step (s) (GridAction.U) - (s2, r) <- WindyGrid.step (s) (GridAction.D) + (s1, r) <- windyGrid.step (s) (GridAction.U) + (s2, r) <- windyGrid.step (s) (GridAction.D) yield s1._1 == s.x && s2._1 == s.x } property ("If wind, R affects both x and y unless in the y's upper bound") = forAll { (s: GridState) => - for (s1, r) <- WindyGrid.step (s) (GridAction.R) + for (s1, r) <- windyGrid.step (s) (GridAction.R) yield (s.x >= 4 && s.x <= 8 && s.y <= 6) ==> (s1._1 != s.x && s1._2 != s.y) } property ("If wind, L affects both x and y unless in the y's upper bound") = forAll { (s: GridState) => - for (s1, r) <- WindyGrid.step (s) (GridAction.L) + for (s1, r) <- windyGrid.step (s) (GridAction.L) yield (s.x >= 4 && s.x <= 8 && s.y <= 6) ==> (s1._1 != s.x && s1._2 != s.y) } // The test runs 1 episode regardless the last argument value (`episodes`) // The fixture has to be setup here, as otherwise the parallelization seems // to rerun training multiple times (if it is put inside the property) - val sarsa = ConcreteSarsa (WindyGrid, 0.1, 0.5, 0.1, 1) - val initials = Randomized.eachOf(WindyGrid.instances.allObservableStates*) - val qs = sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q](), initials).take(5).toList + val sarsa = ConcreteSarsa (windyGrid, 0.1, 0.5, 0.1, 1) + val initials = LazyList (windyGrid.instances.allObservableStates*) + val qs = sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q] (), initials) + .sample (5) import sarsa.vf.apply property ("Q-table values are non-positive") = From 175280eb63c1476a444cb784ad15ca5ac6899df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 08:39:11 +0100 Subject: [PATCH 07/32] Randomized2: Remove repeat (not needed in Randomized2) --- src/main/scala/symsim/concrete/Randomized2.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/scala/symsim/concrete/Randomized2.scala b/src/main/scala/symsim/concrete/Randomized2.scala index 01fdc28f..6db66ac0 100644 --- a/src/main/scala/symsim/concrete/Randomized2.scala +++ b/src/main/scala/symsim/concrete/Randomized2.scala @@ -33,11 +33,6 @@ object Randomized2: def oneOf[A] (choices: A*): Randomized2[A] = probula.Uniform(choices*) - /** For this representation of Randomized, repeat does nothing (an identity), - * we should probably remove repeat from the API. */ - def repeat[A] (ra: =>Randomized2[A]): Randomized2[A] = ra - - given randomizedIsMonad: cats.Monad[Randomized2] = new cats.Monad[Randomized2]: def flatMap[A, B](fa: Randomized2[A])(f: A => Randomized2[B]): Randomized2[B] = fa.flatMap(f) From c7dd99d76f285ebc948f574e34b8cad32c3ac7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 08:40:05 +0100 Subject: [PATCH 08/32] simplemaze: Make the experiments a notch faster for testing (less episodes) --- .../symsim/examples/concrete/simplemaze/SarsaExperiments.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala b/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala index c614b482..2c8a8bd2 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala @@ -18,7 +18,7 @@ class SarsaExperiments alpha = 0.1, gamma = 1, epsilon0 = 0.1, - episodes = 1000000, + episodes = 300000, ) s"SimpleMaze experiment with ${sarsa}" in { From c459a80d9b5205d6fe3fb42c160f94f4d53dbe02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 08:40:26 +0100 Subject: [PATCH 09/32] golf: Fix a broken import --- src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala b/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala index 07cc0570..7b3dddae 100644 --- a/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala +++ b/src/test/scala/symsim/examples/concrete/golf/GolfSpec.scala @@ -7,7 +7,7 @@ import org.scalacheck.Prop.* import symsim.CanTestIn.given import symsim.concrete.ConcreteSarsa -import symsim.concrete.Randomized +import symsim.concrete.Randomized2 // Eliminate the warning on GolfSpec until scalacheck marks Properties it open import scala.language.adhocExtensions @@ -46,7 +46,7 @@ class GolfSpec } property ("Using club D in the sand is the best trained action after 100 episodes") = - val initials = Randomized.repeat(Randomized.const(golf.StartState)).take(200) - val q = sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q](), initials) + val initials = Randomized2.const (golf.StartState).sample (200) + val q = sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q] (), initials) q.sample (50) .forall { (q, _) => sarsa.vf.bestAction (q) (golf.SandState)._1 == Club.D } From e69ae2f550eac383f15c94d18fba603e25a19b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 08:51:57 +0100 Subject: [PATCH 10/32] build: Bump up scalatest to reduce warnings --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 3bf9dcf1..9417580c 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "symsim" ThisBuild / scalaVersion := "3.3.1" -val scalatestVersion = "3.2.14" +val scalatestVersion = "3.2.17" val catsVersion = "2.6.1" From 5da9e44b5413818ff0a5f92375d729e9e5c0f325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 08:56:02 +0100 Subject: [PATCH 11/32] car: Do not use min/max infix as Scala 3 generates warnings against it now --- .../scala/symsim/examples/concrete/braking/CarSpec.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala b/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala index e11e2f61..72fc9759 100644 --- a/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala +++ b/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala @@ -41,16 +41,16 @@ object CarSpec property ("Reward is valid 1") = forAll { (s1: CarState, s2: CarState, a: CarAction) => for - (_, r1) <- car.step (CarState (v = s1.v, p = s1.p min s2.p)) (a) - (_, r2) <- car.step (CarState (v = s1.v, p = s1.p max s2.p)) (a) + (_, r1) <- car.step (CarState (v = s1.v, p = s1.p.min (s2.p))) (a) + (_, r2) <- car.step (CarState (v = s1.v, p = s1.p.max (s2.p))) (a) yield r1 >= r2 } property ("Reward is valid 2") = forAll { (s1: CarState, s2: CarState, a: CarAction) => for - (_, r1) <- car.step (CarState (v = s1.v min s2.v, p = s1.p)) (a) - (_, r2) <- car.step (CarState (v = s1.v max s2.v, p = s1.p)) (a) + (_, r1) <- car.step (CarState (v = s1.v.min (s2.v), p = s1.p)) (a) + (_, r2) <- car.step (CarState (v = s1.v.max (s2.v), p = s1.p)) (a) yield r1 >= r2 } From f7e76c9a84bb78bd23de765659e64e06980c6111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 09:21:19 +0100 Subject: [PATCH 12/32] laws: Abstract away the random seed outside of the laws --- .../examples/concrete/simplebandit/Bandit.scala | 2 +- .../scala/symsim/laws/ConcreteExpectedSarsaLaws.scala | 11 ++++------- src/main/scala/symsim/laws/ConcreteSarsaLaws.scala | 5 +++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala b/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala index 89a836ce..e90c551b 100644 --- a/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala +++ b/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala @@ -62,7 +62,7 @@ class BanditInstances (banditReward: List [Randomized2[BanditReward]]) (using pr BoundedEnumerableFromList (List.range(0, banditReward.size)*) given enumState: BoundedEnumerable[BanditState] = - BoundedEnumerableFromList (false, true) + BoundedEnumerableFromList (false, true) given schedulerIsMonad: Monad[Randomized2] = Randomized2.randomizedIsMonad diff --git a/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala b/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala index 0166ded3..a9a6090e 100644 --- a/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala +++ b/src/main/scala/symsim/laws/ConcreteExpectedSarsaLaws.scala @@ -14,10 +14,6 @@ import symsim.concrete.ConcreteExactRL import symsim.concrete.ConcreteQTable import symsim.concrete.Randomized2 -given spire.random.rng.SecureJava = - spire.random.rng.SecureJava.apply - - /** Laws that have to be obeyed by any refinement of symsim.ConcreetSarsa * * @param sarsa A problem configured together with a learning @@ -40,7 +36,8 @@ given spire.random.rng.SecureJava = case class ConcreteExpectedSarsaLaws[State, ObservableState, Action] (sarsa: ConcreteExactRL[State, ObservableState, Action], gamma: Double - ) extends org.typelevel.discipline.Laws: + ) (using probula.RNG) + extends org.typelevel.discipline.Laws: import sarsa.{agent,vf} import sarsa.agent.instances.given @@ -59,7 +56,7 @@ case class ConcreteExpectedSarsaLaws[State, ObservableState, Action] given Arbitrary[Q] = Arbitrary (vf.genVF (using agent.instances.arbitraryReward)) - + val laws: RuleSet = SimpleRuleSet ( "concreteExpectedSarsa", @@ -71,7 +68,7 @@ case class ConcreteExpectedSarsaLaws[State, ObservableState, Action] val n = 4000 val ε = 0.1 // Ignore ε in the problem as it might be zero for // the sake of the other test - + val trials = for s_t <- agent.initialize a_tt <- vf.chooseAction (ε) (q) (agent.observe (s_t)) diff --git a/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala b/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala index 8640fabd..1b17eb32 100644 --- a/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala +++ b/src/main/scala/symsim/laws/ConcreteSarsaLaws.scala @@ -35,7 +35,8 @@ import symsim.concrete.Randomized2 case class ConcreteSarsaLaws[State, ObservableState, Action] (sarsa: ConcreteExactRL[State, ObservableState, Action], gamma: Double - ) extends org.typelevel.discipline.Laws: + ) (using probula.RNG) + extends org.typelevel.discipline.Laws: import sarsa.{agent,vf} import sarsa.agent.instances.given @@ -116,7 +117,7 @@ case class ConcreteSarsaLaws[State, ObservableState, Action] // A random variable representing differences between updates - val diffs = (sutUpdates zip specUpdates) + val diffs = sutUpdates.zip (specUpdates) .map { _ - _ } // Infer the posterior on mean update. From 5fac4705e902393057d2dda899fffd71c1d9f96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 09:22:24 +0100 Subject: [PATCH 13/32] gitignore: Ignore the csv outputs from experiments --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7ea58b9d..448c0e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ project/target/ .bsp/ pump.policy pump.q +*.csv From 06379f230c38f14101aa68019453e4a8e8e7b5ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 09:51:46 +0100 Subject: [PATCH 14/32] probula: Sanitize where the rng state is created It should only be created in top-level executable files (which in this project means Spec files) --- .../scala/symsim/concrete/BdlConcreteExpectedSarsa.scala | 3 ++- src/main/scala/symsim/concrete/BdlConcreteSarsa.scala | 3 ++- src/main/scala/symsim/concrete/ConcreteExactRL.scala | 9 ++++----- .../scala/symsim/concrete/ConcreteExpectedSarsa.scala | 3 ++- src/main/scala/symsim/concrete/ConcreteQLearning.scala | 3 ++- .../symsim/concrete/ConcreteQLearningWithDecay.scala | 3 ++- src/main/scala/symsim/concrete/ConcreteSarsa.scala | 3 ++- src/main/scala/symsim/concrete/Randomized2.scala | 2 +- .../symsim/examples/concrete/simplebandit/Bandit.scala | 2 +- src/test/scala/symsim/ExperimentSpec.scala | 7 +++---- .../BdlConcreteExpectedSarsaIsExpectedSarsa.scala | 2 ++ .../examples/concrete/braking/CarIsAgentSpec.scala | 8 ++++++-- .../symsim/examples/concrete/cartpole/Experiments.scala | 6 +++--- .../examples/concrete/cliffwalking/Experiments.scala | 6 ++++-- .../examples/concrete/mountaincar/Experiments.scala | 6 ++++-- .../symsim/examples/concrete/pumping/Experiments.scala | 5 ++++- .../examples/concrete/simplebandit/Experiments.scala | 3 +++ .../examples/concrete/simplemaze/SarsaExperiments.scala | 2 +- .../examples/concrete/windygrid/WindyGridSpec.scala | 3 ++- 19 files changed, 50 insertions(+), 29 deletions(-) diff --git a/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala b/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala index 3a175411..3cd56a69 100644 --- a/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala +++ b/src/main/scala/symsim/concrete/BdlConcreteExpectedSarsa.scala @@ -13,7 +13,8 @@ case class BdlConcreteExpectedSarsa [ val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends BdlLearn[State, ObservableState, Action, Double, Randomized2], +) (using val rng: probula.RNG) + extends BdlLearn[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala b/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala index 1fbad143..4ef45a38 100644 --- a/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala +++ b/src/main/scala/symsim/concrete/BdlConcreteSarsa.scala @@ -13,7 +13,8 @@ case class BdlConcreteSarsa [ val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends BdlLearn[State, ObservableState, Action, Double, Randomized2], +) (using val rng: probula.RNG) + extends BdlLearn[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteExactRL.scala b/src/main/scala/symsim/concrete/ConcreteExactRL.scala index a75e4847..5ff51fa2 100644 --- a/src/main/scala/symsim/concrete/ConcreteExactRL.scala +++ b/src/main/scala/symsim/concrete/ConcreteExactRL.scala @@ -3,9 +3,6 @@ package concrete import cats.kernel.BoundedEnumerable -given spire.random.rng.SecureJava = - spire.random.rng.SecureJava.apply - trait ConcreteExactRL[State, ObservableState, Action] extends ExactRL[State, ObservableState, Action, Double, Randomized2]: @@ -24,10 +21,12 @@ trait ConcreteExactRL[State, ObservableState, Action] // TODO: unclear if this is general (if it turns out to be the same im // symbolic or approximate algos we should promote this to the trait + + given rng: probula.RNG def runQ: (Q, List[Q]) = - val initials = agent.initialize.sample(episodes) - val outcome = learn (vf.initialize, List[VF](), initials).sample() + val initials = agent.initialize.sample (episodes) + val outcome = learn (vf.initialize, List[VF] (), initials).sample () (outcome._1, outcome._2) override def run: Policy = diff --git a/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala b/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala index 579626ad..eeff03a7 100644 --- a/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala +++ b/src/main/scala/symsim/concrete/ConcreteExpectedSarsa.scala @@ -7,7 +7,8 @@ case class ConcreteExpectedSarsa[State, ObservableState, Action] ( val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends ExpectedSarsa[State, ObservableState, Action, Double, Randomized2], +) (using val rng: probula.RNG) + extends ExpectedSarsa[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteQLearning.scala b/src/main/scala/symsim/concrete/ConcreteQLearning.scala index ed9c722a..833adb98 100644 --- a/src/main/scala/symsim/concrete/ConcreteQLearning.scala +++ b/src/main/scala/symsim/concrete/ConcreteQLearning.scala @@ -13,7 +13,8 @@ case class ConcreteQLearning [ val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends QLearning[State, ObservableState, Action, Double, Randomized2], +) (using val rng: probula.RNG) + extends QLearning[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala b/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala index 34c1707a..cddb0e57 100644 --- a/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala +++ b/src/main/scala/symsim/concrete/ConcreteQLearningWithDecay.scala @@ -15,7 +15,8 @@ case class ConcreteQLearningWithDecay [ val epsilon0: Probability, val episodes: Int, -) extends QLearning[State, ObservableState, Action, Double, Randomized2], +) (using val rng: probula.RNG) + extends QLearning[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], BoundedEpsilonDecay: diff --git a/src/main/scala/symsim/concrete/ConcreteSarsa.scala b/src/main/scala/symsim/concrete/ConcreteSarsa.scala index 7afc355f..0b7528aa 100644 --- a/src/main/scala/symsim/concrete/ConcreteSarsa.scala +++ b/src/main/scala/symsim/concrete/ConcreteSarsa.scala @@ -13,7 +13,8 @@ case class ConcreteSarsa [ val gamma: Double, val epsilon0: Probability, val episodes: Int, -) extends Sarsa[State, ObservableState, Action, Double, Randomized2], +) (using val rng: probula.RNG) + extends Sarsa[State, ObservableState, Action, Double, Randomized2], ConcreteExactRL[State, ObservableState, Action], NoDecay: diff --git a/src/main/scala/symsim/concrete/Randomized2.scala b/src/main/scala/symsim/concrete/Randomized2.scala index 6db66ac0..54828efe 100644 --- a/src/main/scala/symsim/concrete/Randomized2.scala +++ b/src/main/scala/symsim/concrete/Randomized2.scala @@ -95,7 +95,7 @@ object Randomized2: /** Perform an imperative operation that depends on one sample from this * Randomized. This is mostly meant for IO at this point. */ - def run (f: A => Unit): Unit = f(self.sample ()) + def run (f: A => Unit) (using RNG): Unit = f(self.sample ()) def filter (p: A => Boolean): Randomized2[A] = self.filter (p) diff --git a/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala b/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala index e90c551b..ea28d093 100644 --- a/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala +++ b/src/main/scala/symsim/examples/concrete/simplebandit/Bandit.scala @@ -59,7 +59,7 @@ class BanditInstances (banditReward: List [Randomized2[BanditReward]]) (using pr extends AgentConstraints[BanditState, BanditState, BanditAction, BanditReward, Randomized2]: given enumAction: BoundedEnumerable[BanditAction] = - BoundedEnumerableFromList (List.range(0, banditReward.size)*) + BoundedEnumerableFromList (List.range (0, banditReward.size)*) given enumState: BoundedEnumerable[BanditState] = BoundedEnumerableFromList (false, true) diff --git a/src/test/scala/symsim/ExperimentSpec.scala b/src/test/scala/symsim/ExperimentSpec.scala index 7219fc38..3668ddc7 100644 --- a/src/test/scala/symsim/ExperimentSpec.scala +++ b/src/test/scala/symsim/ExperimentSpec.scala @@ -10,8 +10,6 @@ import symsim.concrete.Randomized2 import cats.syntax.all.* import symsim.concrete.Randomized2.* -given spire.random.rng.SecureJava = - spire.random.rng.SecureJava.apply trait ExperimentSpec[State, ObservableState, Action] extends org.scalatest.freespec.AnyFreeSpec, @@ -70,12 +68,13 @@ trait ExperimentSpec[State, ObservableState, Action] policies: List[setup.Policy], initials: Option[Randomized2[State]] = None, noOfEpisodes: Int = 5 - ): EvaluationResults = + ) (using probula.RNG): EvaluationResults = val ss: Randomized2[State] = initials.getOrElse (setup.agent.initialize) for p <- policies episodeRewards: Randomized2[Randomized2[Double]] = setup.evaluate (p, ss) - rewards: Randomized2[Double] = episodeRewards.map { e => e.sample () } + rewards: Randomized2[Double] = + episodeRewards.map { e => e.sample () } yield rewards.sample (noOfEpisodes).toList diff --git a/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala b/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala index 8b04a5b6..4603d767 100644 --- a/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala +++ b/src/test/scala/symsim/concrete/BdlConcreteExpectedSarsaIsExpectedSarsa.scala @@ -5,6 +5,8 @@ import symsim.examples.concrete.mountaincar.MountainCar private val mountainCar = new MountainCar (using spire.random.rng.SecureJava.apply) +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply import mountainCar.instances.given diff --git a/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala b/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala index 1cb2e884..e2aeea65 100644 --- a/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala +++ b/src/test/scala/symsim/examples/concrete/braking/CarIsAgentSpec.scala @@ -4,8 +4,12 @@ package examples.concrete.braking import laws.AgentLaws import laws.EpisodicLaws +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val car = new Car + class CarIsAgentSpec extends SymSimSpec: - checkAll ("concrete.braking.Car is an Agent", AgentLaws (new Car).laws) - checkAll ("concrete.braking.Car is Episodic", EpisodicLaws (new Car).laws) + checkAll ("concrete.braking.Car is an Agent", AgentLaws (car).laws) + checkAll ("concrete.braking.Car is Episodic", EpisodicLaws (car).laws) diff --git a/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala b/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala index 374e64ae..6b578f57 100644 --- a/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala @@ -1,9 +1,9 @@ package symsim package examples.concrete.cartpole -// Import evidence that states and actions can be enumerated -private val cartPole = - new CartPole (using spire.random.rng.SecureJava.apply) +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val cartPole: CartPole = new CartPole import cartPole.instances.{enumAction, enumState} class Experiments extends diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala index ddbc62ba..a5a44568 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala @@ -1,8 +1,10 @@ package symsim package examples.concrete.cliffWalking -private val cliffWalking = - new CliffWalking (using spire.random.rng.SecureJava.apply) + +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val cliffWalking: CliffWalking = new CliffWalking import cliffWalking.instances.{enumAction, enumState} class Experiments diff --git a/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala b/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala index 99ba079e..eea19962 100644 --- a/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala @@ -1,8 +1,10 @@ package symsim package examples.concrete.mountaincar -private val mountainCar = - new MountainCar (using spire.random.rng.SecureJava.apply) + +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val mountainCar: MountainCar = new MountainCar import mountainCar.instances.{enumAction, enumState} class Experiments diff --git a/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala b/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala index 4f12e6d8..8ace3b67 100644 --- a/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/pumping/Experiments.scala @@ -1,7 +1,10 @@ package symsim package examples.concrete.pumping -private val pump = new Pump (using spire.random.rng.SecureJava.apply) + +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val pump: Pump = new Pump class Experiments extends ExperimentSpec[PumpState, ObservablePumpState, PumpAction]: diff --git a/src/test/scala/symsim/examples/concrete/simplebandit/Experiments.scala b/src/test/scala/symsim/examples/concrete/simplebandit/Experiments.scala index aa48e901..634162dd 100644 --- a/src/test/scala/symsim/examples/concrete/simplebandit/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/simplebandit/Experiments.scala @@ -1,6 +1,9 @@ package symsim package examples.concrete.simplebandit +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply + class Experiments extends ExperimentSpec[BanditState,BanditState,BanditAction]: diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala b/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala index 2c8a8bd2..26c5d6d8 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/SarsaExperiments.scala @@ -1,7 +1,7 @@ package symsim package examples.concrete.simplemaze -given spire.random.rng.SecureJava = +private given spire.random.rng.SecureJava = spire.random.rng.SecureJava.apply private val maze = new Maze diff --git a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala index c152d988..cb651838 100644 --- a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala +++ b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala @@ -10,7 +10,8 @@ import symsim.concrete.ConcreteSarsa import symsim.concrete.Randomized2 import symsim.concrete.Randomized2.given -given spire.random.rng.SecureJava = spire.random.rng.SecureJava.apply +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply private val windyGrid = new WindyGrid From 9f89ca402e0d682ea7c7b4b94f25ec37861455bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 13:20:41 +0100 Subject: [PATCH 15/32] Randomized2: Edit whitespace to be consistent with the rest of the project --- .../scala/symsim/concrete/Randomized2.scala | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/main/scala/symsim/concrete/Randomized2.scala b/src/main/scala/symsim/concrete/Randomized2.scala index 54828efe..d19e2893 100644 --- a/src/main/scala/symsim/concrete/Randomized2.scala +++ b/src/main/scala/symsim/concrete/Randomized2.scala @@ -2,7 +2,7 @@ package symsim.concrete import scala.annotation.targetName import org.scalacheck.{Gen, Prop} -import probula.{Dist, RNG} +import probula.{Dist, Name, IData} opaque type Randomized2[+A] = Dist[A] @@ -16,42 +16,42 @@ object Randomized2: probula.Dirac (a) def prob: Randomized2[Double] = - probula.UniformC(0.0, 1.0) + probula.UniformC (0.0, 1.0) def between (minInclusive: Int, maxExclusive: Int): Randomized2[Int] = - probula.Uniform(minInclusive, maxExclusive+1) + probula.Uniform (minInclusive, maxExclusive+1) def between (minInclusive: Double, maxExclusive: Double): Randomized2[Double] = - probula.UniformC(minInclusive, maxExclusive) + probula.UniformC (minInclusive, maxExclusive) def gaussian (mean: Double = 0.0, stdDev: Double = 1.0): Randomized2[Double] = - probula.Gaussian(mean, stdDev) + probula.Gaussian (mean, stdDev) def coin (bias: Probability): Randomized2[Boolean] = - probula.Bernoulli(bias) + probula.Bernoulli (bias) def oneOf[A] (choices: A*): Randomized2[A] = - probula.Uniform(choices*) + probula.Uniform (choices*) given randomizedIsMonad: cats.Monad[Randomized2] = new cats.Monad[Randomized2]: - def flatMap[A, B](fa: Randomized2[A])(f: A => Randomized2[B]): Randomized2[B] = + def flatMap[A, B] (fa: Randomized2[A])(f: A => Randomized2[B]): Randomized2[B] = fa.flatMap(f) - def pure[A](x: A): Randomized2[A] = + def pure[A] (x: A): Randomized2[A] = probula.Dirac[A] (x) - import probula.{Name, IData, RNG} + def tailRecM[A, B] (ini: A) (f: A => Randomized2[Either[A, B]]) + : Randomized2[B] = new Dist[B]: + def name: Name = Name.Suffixed(f (ini).name, "tailRecM") + def sample[C >: B] (using probula.RNG): IData[C] = + val newChain = summon[cats.Monad[LazyList]] + .tailRecM[A, B] (ini) { a => f(a).sample.chain } + IData (name, newChain) - def tailRecM[A, B](ini: A)(f: A => Randomized2[Either[A, B]]): Randomized2[B] = - val a0 = f(ini) - new Dist[B]: - def name: Name = Name.Suffixed(a0.name, "tailRecM") - def sample[C >: B](using RNG): IData[C] = - val newChain = summon[cats.Monad[LazyList]] - .tailRecM[A, B](ini) { a => f(a).sample.chain } - IData(name, newChain) - given canTestInRandomized (using RNG): symsim.CanTestIn[Randomized2] = - new symsim.CanTestIn[Randomized2] { + + + given canTestInRandomized (using probula.RNG): symsim.CanTestIn[Randomized2] = + new symsim.CanTestIn[Randomized2]: @targetName ("toPropBoolean") def toProp (rProp: Randomized2[Boolean]) = @@ -64,9 +64,8 @@ object Randomized2: // state to objects? def toGen[A] (ra: => Randomized2[A]): Gen[A] = val list: LazyList[A] = ra.sample.chain - Gen.choose(0, 1000) + Gen.choose (0, 1000) .map { i => list (i) } - } /** This extensions should ideally be used at a top-level of the program, @@ -77,8 +76,10 @@ object Randomized2: * always the same if you call several times. * (at least in the current implementation) */ - def sample () (using RNG): A = - self.sample(1)(using summon[RNG]).chain.head + def sample () (using rng: probula.RNG): A = + self.sample (1) (using rng) + .chain + .head /** Get n samples from randomized. Note that the sample will be random but * always the same if you call several times. @@ -89,13 +90,16 @@ object Randomized2: * not have n samples. * */ - def sample (n: Int) (using RNG): LazyList[A] = - self.sample(n).chain.take(n) + def sample (n: Int) (using probula.RNG): LazyList[A] = + self.sample (n) + .chain + .take (n) /** Perform an imperative operation that depends on one sample from this * Randomized. This is mostly meant for IO at this point. */ - def run (f: A => Unit) (using RNG): Unit = f(self.sample ()) + def run (f: A => Unit) (using probula.RNG): Unit = + f (self.sample ()) def filter (p: A => Boolean): Randomized2[A] = self.filter (p) From e58640e9a06e54444d8e38cb831d2c6036a81e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 14:23:57 +0100 Subject: [PATCH 16/32] Gaussian simple bandit: Minor cleanup --- .../concrete/simplebandit/BanditObjectGaussian.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala b/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala index c0be5511..97627539 100644 --- a/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala +++ b/src/test/scala/symsim/examples/concrete/simplebandit/BanditObjectGaussian.scala @@ -1,9 +1,8 @@ -package symsim -package examples.concrete.simplebandit +package symsim.examples.concrete.simplebandit import symsim.concrete.Randomized2 -val BanditObjGaussian = new Bandit (List ( +val BanditObjGaussian: Bandit = new Bandit (List ( Randomized2.gaussian (-0.2, 1), Randomized2.gaussian ( 0.2, 1), Randomized2.gaussian ( 0.4, 1), From 0391a5ac85f8f65747327a6d72d1971fcd89dce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 14:24:09 +0100 Subject: [PATCH 17/32] cartpole: Fix an assertion bug --- .../examples/concrete/cartpole/CartPole.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala b/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala index 1bf97fb3..0140690b 100644 --- a/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala +++ b/src/main/scala/symsim/examples/concrete/cartpole/CartPole.scala @@ -21,14 +21,14 @@ abstract class CartPoleAbstractState ( val pv: Double ): - require (cp >= CpMin, "cp too low") - require (cp <= CpMax, "cp too high") - require (cv >= CvMin, "cv too low") - require (cv <= CvMax, "cv too high") - require (pa >= PaMin, "pa too low") - require (pa <= PaMax, "pa too high") - require (pv >= PvMin, "pv too low") - require (pv <= PvMin, "pv too high") + require (cp >= CpMin, s"cp too low: ¬ ($pv ≥ $CpMin)") + require (cp <= CpMax, s"cp too high: ¬ ($pv ≤ $CpMax)") + require (cv >= CvMin, s"cv too low: ¬ ($pv ≥ $CvMin)") + require (cv <= CvMax, s"cv too high: ¬ ($pv ≤ $CvMax)") + require (pa >= PaMin, s"pa too low: ¬ ($pv ≥ $PaMin)") + require (pa <= PaMax, s"pa too high: ¬ ($pv ≤ $PaMax)") + require (pv >= PvMin, s"pv too low: ¬ ($pv ≥ $PvMin)") + require (pv <= PvMax, s"pv too high: ¬ ($pv ≤ $PvMax)") override def toString: String = s"[cart position=$cp, cart velocity=$cv," From 72214de6b4d916b377165a5f1f521a284a488226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 14:36:29 +0100 Subject: [PATCH 18/32] cartpole: Reduce the number of episodes to save memory - otherwise we have OOM errors --- .../scala/symsim/examples/concrete/cartpole/Experiments.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala b/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala index 6b578f57..fb4c241f 100644 --- a/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cartpole/Experiments.scala @@ -14,7 +14,7 @@ class Experiments extends alpha = 0.1, gamma = 0.1, epsilon0 = 0.05, - episodes = 20000, + episodes = 2000, ) s"CartPole experiment with $sarsa" in { From c66ff2af1e94a3afdcd8274073ddc0dd5d437bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 15:02:21 +0100 Subject: [PATCH 19/32] cliffwalking: Move the state invariant test to the state So that we can diagnostic information in more situations, not only when moving (it was previously placed in move) --- .../concrete/cliffwalking/CliffWalking.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala index 1f1c08b0..345de362 100644 --- a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala +++ b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala @@ -9,9 +9,19 @@ import org.scalacheck.{Arbitrary, Gen} import symsim.concrete.Randomized2 +val BoardWidth: Int = 11 +val BoardHeight: Int = 3 + case class CWState (x: Int, y: Int): + require (x >= 0, s"Negative horizontal position $x") + require (x <= BoardWidth, s"Out-Of-Width x: ¬($x ≤ $BoardWidth)") + require (y >= 0, s"Negative vertical board position $y}") + require (y <= BoardHeight, s"Out-of-Height y: ¬($y ≤ $BoardHeight)") + override def toString: String = s"($x,$y)" + + type CWObservableState = CWState type CWReward = Double @@ -24,9 +34,6 @@ enum CWAction: case CWAction.Left => "←" case CWAction.Right => "→" -val BoardWidth: Int = 11 -val BoardHeight: Int = 3 - class CliffWalking (using probula.RNG) extends Agent[CWState, CWObservableState, CWAction, CWReward, Randomized2] with Episodic: @@ -44,10 +51,6 @@ class CliffWalking (using probula.RNG) def observe (s: CWState): CWObservableState = s def move (s: CWState, a: CWAction): CWState = - require (s.x >= 0) - require (s.x <= BoardWidth) - require (s.y >= 0) - require (s.y <= BoardHeight) a match case CWAction.Up => CWState (s.x, (s.y + 1).min (BoardHeight)) From 2159d7c9b8617c78533f3f768d710d69931b5391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 15:10:44 +0100 Subject: [PATCH 20/32] Randomized2: fix a bug in integer between It was not excluding the right point margin as the spec would indicate --- src/main/scala/symsim/concrete/Randomized2.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/symsim/concrete/Randomized2.scala b/src/main/scala/symsim/concrete/Randomized2.scala index d19e2893..9bca9d3d 100644 --- a/src/main/scala/symsim/concrete/Randomized2.scala +++ b/src/main/scala/symsim/concrete/Randomized2.scala @@ -19,10 +19,10 @@ object Randomized2: probula.UniformC (0.0, 1.0) def between (minInclusive: Int, maxExclusive: Int): Randomized2[Int] = - probula.Uniform (minInclusive, maxExclusive+1) + probula.Uniform (minInclusive, maxExclusive - 1) - def between (minInclusive: Double, maxExclusive: Double): Randomized2[Double] = - probula.UniformC (minInclusive, maxExclusive) + def between (minInclusive: Double, maxInclusive: Double): Randomized2[Double] = + probula.UniformC (minInclusive, maxInclusive) def gaussian (mean: Double = 0.0, stdDev: Double = 1.0): Randomized2[Double] = probula.Gaussian (mean, stdDev) From e8586f928d6d062893c1b37e7bf2c7391bba891b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 15:17:47 +0100 Subject: [PATCH 21/32] cliffwalking: A slight bug in initializing the state It seems that it was forgetting initializing at the max edges of the board --- .../symsim/examples/concrete/cliffwalking/CliffWalking.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala index 345de362..dbb7e6c3 100644 --- a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala +++ b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala @@ -9,6 +9,7 @@ import org.scalacheck.{Arbitrary, Gen} import symsim.concrete.Randomized2 + val BoardWidth: Int = 11 val BoardHeight: Int = 3 @@ -70,8 +71,8 @@ class CliffWalking (using probula.RNG) Randomized2.const (s1, cwReward (s1) (a)) def initialize: Randomized2[CWState] = { for - x <- Randomized2.between (0, BoardWidth) - y <- Randomized2.between (0, BoardHeight) + x <- Randomized2.between (0, BoardWidth +1) + y <- Randomized2.between (0, BoardHeight+1) s = CWState (x, y) yield s }.filter { !this.isFinal (_) } From 5605f6386cf6b84ce8a680b563903b72ae488598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 15:19:49 +0100 Subject: [PATCH 22/32] Car: Remove warning on using min/max infix --- src/main/scala/symsim/examples/concrete/braking/Car.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/symsim/examples/concrete/braking/Car.scala b/src/main/scala/symsim/examples/concrete/braking/Car.scala index c9012137..bd68f01f 100644 --- a/src/main/scala/symsim/examples/concrete/braking/Car.scala +++ b/src/main/scala/symsim/examples/concrete/braking/Car.scala @@ -48,7 +48,7 @@ class Car (using probula.RNG) val dp = (s.p/5.0).floor * 5.0 val dv = (s.v/5.0).floor * 5.0 - CarState (dv min 10.0, dp min 15.0) + CarState (dv.min (10.0), dp.min (15.0)) private def carReward (s: CarState) (a: CarAction): CarReward = From de2c9a313b3d8403634b7918126a91375bc26780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 20:27:11 +0100 Subject: [PATCH 23/32] minor clean up in various places --- .../examples/concrete/braking/Car.scala | 4 ++-- .../concrete/cliffwalking/CliffWalking.scala | 20 +++++++++---------- .../concrete/windygrid/WindyGrid.scala | 14 +++++++++---- .../concrete/braking/Experiments.scala | 4 +++- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/braking/Car.scala b/src/main/scala/symsim/examples/concrete/braking/Car.scala index bd68f01f..7e96eb61 100644 --- a/src/main/scala/symsim/examples/concrete/braking/Car.scala +++ b/src/main/scala/symsim/examples/concrete/braking/Car.scala @@ -69,11 +69,11 @@ class Car (using probula.RNG) Randomized2.const (s1, carReward (s1) (a)) - def initialize: Randomized2[CarState] = (for + def initialize: Randomized2[CarState] = { for v <- Randomized2.between (0.0, 15.0) p <- Randomized2.between (0.0, 20.0) s = CarState (v, p) - yield s).filter { !this.isFinal (_)} + yield s }.filter { !this.isFinal (_) } end Car diff --git a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala index dbb7e6c3..5fe80270 100644 --- a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala +++ b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala @@ -51,13 +51,11 @@ class CliffWalking (using probula.RNG) def observe (s: CWState): CWObservableState = s - def move (s: CWState, a: CWAction): CWState = - - a match - case CWAction.Up => CWState (s.x, (s.y + 1).min (BoardHeight)) - case CWAction.Down => CWState (s.x, (s.y - 1).max (0)) - case CWAction.Right => CWState ((s.x + 1).min (BoardWidth), s.y) - case CWAction.Left => CWState ((s.x - 1).max (0), s.y) + def move (s: CWState, a: CWAction): CWState = a match + case CWAction.Up => CWState (s.x, (s.y + 1).min (BoardHeight)) + case CWAction.Down => CWState (s.x, (s.y - 1).max (0)) + case CWAction.Right => CWState ((s.x + 1).min (BoardWidth), s.y) + case CWAction.Left => CWState ((s.x - 1).max (0), s.y) def cwReward (s: CWState) (a: CWAction): CWReward = @@ -71,8 +69,8 @@ class CliffWalking (using probula.RNG) Randomized2.const (s1, cwReward (s1) (a)) def initialize: Randomized2[CWState] = { for - x <- Randomized2.between (0, BoardWidth +1) - y <- Randomized2.between (0, BoardHeight+1) + x <- Randomized2.between (0, BoardWidth + 1) + y <- Randomized2.between (0, BoardHeight + 1) s = CWState (x, y) yield s }.filter { !this.isFinal (_) } @@ -91,8 +89,8 @@ class CliffWalkingInstances (using probula.RNG) given enumState: BoundedEnumerable[CWObservableState] = val ss = for - x <- (0 to BoardWidth).toSeq - y <- (0 to BoardHeight).toSeq + x <- 0 to BoardWidth + y <- 0 to BoardHeight yield CWState (x, y) BoundedEnumerableFromList (ss*) diff --git a/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala b/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala index b86ba548..3997a778 100644 --- a/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala +++ b/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala @@ -10,9 +10,15 @@ import org.scalacheck.{Arbitrary, Gen} import symsim.concrete.Randomized2 +val xMin = 1 +val xMax = 10 +val yMin = 1 +val yMax = 7 + case class GridState (x: Int, y: Int): + require (x >= 1 && x <= 10 && y >= 1 && y <= 7) override def toString: String = s"($x,$y)" - + type GridObservableState = GridState type GridReward = Double @@ -67,7 +73,7 @@ class WindyGrid (using probula.RNG) extends x <- Randomized2.between (1, 11) y <- Randomized2.between (1, 8) s = GridState (x, y) - yield s }.filter { ! this.isFinal (_) } + yield s }.filter { !this.isFinal (_) } val instances = new WindyGridInstances @@ -85,8 +91,8 @@ class WindyGridInstances (using probula.RNG) given enumState: BoundedEnumerable[GridObservableState] = val ss = for - x <- Seq (1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - y <- Seq (1, 2, 3, 4, 5, 6, 7) + x <- xMin to xMax + y <- yMin to yMax yield GridState (x, y) BoundedEnumerableFromList (ss*) diff --git a/src/test/scala/symsim/examples/concrete/braking/Experiments.scala b/src/test/scala/symsim/examples/concrete/braking/Experiments.scala index e3d0a42d..ac48ac8b 100644 --- a/src/test/scala/symsim/examples/concrete/braking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/braking/Experiments.scala @@ -3,7 +3,9 @@ package examples.concrete.braking import symsim.concrete.Randomized2 -private val car = new Car (using spire.random.rng.SecureJava.apply) +private given spire.random.rng.SecureJava = + spire.random.rng.SecureJava.apply +private val car: Car = new Car import car.instances.{enumAction, enumState} class Experiments From 91b0bfd9bbe70a8bdea5fe57bd4caebb040e1e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 21:51:05 +0100 Subject: [PATCH 24/32] simplemaze: Fix a bug in initialization + few minor cleanup edits This seems to be an old bug! It was not initializing in the rightmost column. --- .../examples/concrete/simplemaze/Maze.scala | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala b/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala index af4732cc..33fa16f7 100644 --- a/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala +++ b/src/main/scala/symsim/examples/concrete/simplemaze/Maze.scala @@ -47,25 +47,24 @@ enum MazeAction: case Left, Right, Up, Down import MazeAction.* -class Maze (using probula.RNG) - extends - Agent[MazeState, MazeObservableState, MazeAction, MazeReward, Randomized2], - Episodic: +class Maze (using probula.RNG) extends + Agent[MazeState, MazeObservableState, MazeAction, MazeReward, Randomized2], + Episodic: val TimeHorizon: Int = 2000 def isFinal (s: MazeState): Boolean = - (s._1, s._2) == (4, 3) || (s._1, s._2) == (4, 2) || s._3 == TimeHorizon + (s._1, s._2) == (4, 3) || (s._1, s._2) == (4, 2) || s._3 >= TimeHorizon // Maze is discrete def observe (s: MazeState): MazeObservableState = (s._1, s._2) // We are not using the original reward function from AIAMA as it // gives to unstable learning results - private def mazeReward (s: MazeState): MazeReward = (s._1, s._2) match - case (4, 3) => +0.0 // Good final state - case (4, 2) => -1000.0 // Bad final state (dead) - case (_, _) => -1.0 + private def mazeReward (s: MazeState): MazeReward = s match + case (4, 3, _) => +0.0 // Good final state + case (4, 2, _) => -1000.0 // Bad final state (dead) + case (_, _, _) => -1.0 def distort (a: MazeAction): Randomized2[MazeAction] = a match @@ -76,10 +75,10 @@ class Maze (using probula.RNG) def successor (s: MazeState) (a: MazeAction): MazeState = require (valid (s)) val result = a match - case Up => (s._1, s._2+1, s._3+1) - case Down => (s._1, s._2-1, s._3+1) - case Left => (s._1-1, s._2, s._3+1) - case Right => (s._1+1, s._2, s._3+1) + case Up => (s._1, s._2 + 1, s._3 + 1) + case Down => (s._1, s._2 - 1, s._3 + 1) + case Left => (s._1 - 1, s._2, s._3 + 1) + case Right => (s._1 + 1, s._2, s._3 + 1) if valid (result) then result else s val attention = 0.8 @@ -87,15 +86,15 @@ class Maze (using probula.RNG) def step (s: MazeState) (a: MazeAction): Randomized2[(MazeState, MazeReward)] = for precise <- Randomized2.coin (attention) - action <- if precise then Randomized2.const (a) else distort (a) + action <- if precise then Randomized2.const (a) else distort (a) newState = successor (s) (action) yield (newState, mazeReward (newState)) def initialize: Randomized2[MazeState] = { for - x <- Randomized2.between (1, 4) - y <- Randomized2.between (1, 3) - t = 0 - s = (x, y, t) + x <- Randomized2.between (1, 4 + 1) + y <- Randomized2.between (1, 3 + 1) + t = 0 + s = (x, y, t) yield s }.filter { s => !isFinal (s) && valid (s) } val instances = new MazeInstances From 6219ca1a08cd0d637d30e552af2de4001aad7578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 22:00:29 +0100 Subject: [PATCH 25/32] cliffwalking: Fix observable state in cliffwalking --- .../concrete/cliffwalking/CliffWalking.scala | 102 ++++++++++-------- .../concrete/cliffwalking/Experiments.scala | 6 +- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala index 5fe80270..0ee3f15b 100644 --- a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala +++ b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala @@ -13,7 +13,7 @@ import symsim.concrete.Randomized2 val BoardWidth: Int = 11 val BoardHeight: Int = 3 -case class CWState (x: Int, y: Int): +class CWObservableState (x: Int, y: Int): require (x >= 0, s"Negative horizontal position $x") require (x <= BoardWidth, s"Out-Of-Width x: ¬($x ≤ $BoardWidth)") require (y >= 0, s"Negative vertical board position $y}") @@ -21,9 +21,12 @@ case class CWState (x: Int, y: Int): override def toString: String = s"($x,$y)" +case class CWState (x: Int, y: Int, t: Int) + extends CWObservableState (x, y): + + override def toString: String = s"($x,$y,$t)" -type CWObservableState = CWState type CWReward = Double enum CWAction: @@ -44,18 +47,22 @@ class CliffWalking (using probula.RNG) * not actually termintae the episodes. It is a bug if they run * longer. */ - val TimeHorizon: Int = 2000 + val TimeHorizon: Int = 1000 def isFinal (s: CWState): Boolean = - s.y == 0 && s.x > 0 + (s.y == 0 && s.x > 0) || s.t >= TimeHorizon def observe (s: CWState): CWObservableState = s def move (s: CWState, a: CWAction): CWState = a match - case CWAction.Up => CWState (s.x, (s.y + 1).min (BoardHeight)) - case CWAction.Down => CWState (s.x, (s.y - 1).max (0)) - case CWAction.Right => CWState ((s.x + 1).min (BoardWidth), s.y) - case CWAction.Left => CWState ((s.x - 1).max (0), s.y) + case CWAction.Up => + CWState (s.x, (s.y + 1).min (BoardHeight), s.t + 1) + case CWAction.Down => + CWState (s.x, (s.y - 1).max (0), s.t + 1) + case CWAction.Right => + CWState ((s.x + 1).min (BoardWidth), s.y, s.t + 1) + case CWAction.Left => + CWState ((s.x - 1).max (0), s.y, s.t + 1) def cwReward (s: CWState) (a: CWAction): CWReward = @@ -71,47 +78,50 @@ class CliffWalking (using probula.RNG) def initialize: Randomized2[CWState] = { for x <- Randomized2.between (0, BoardWidth + 1) y <- Randomized2.between (0, BoardHeight + 1) - s = CWState (x, y) + s = CWState (x, y, 0) yield s }.filter { !this.isFinal (_) } val instances = new CliffWalkingInstances -end CliffWalking - -/** Here is a proof that our types actually deliver on everything that an Agent - * needs to be able to do to work in the framework. - */ -class CliffWalkingInstances (using probula.RNG) - extends AgentConstraints[CWState, CWObservableState, CWAction, CWReward, Randomized2]: - - given enumAction: BoundedEnumerable[CWAction] = - BoundedEnumerableFromList (CWAction.Up, CWAction.Down, CWAction.Left, CWAction.Right) - - given enumState: BoundedEnumerable[CWObservableState] = - val ss = for - x <- 0 to BoardWidth - y <- 0 to BoardHeight - yield CWState (x, y) - BoundedEnumerableFromList (ss*) - - - given schedulerIsMonad: Monad[Randomized2] = - Randomized2.randomizedIsMonad - - given canTestInScheduler: CanTestIn[Randomized2] = - Randomized2.canTestInRandomized - - lazy val genCWState: Gen[CWState] = for - x <- Gen.oneOf ((0 to BoardWidth).toSeq) - y <- Gen.oneOf ((0 to BoardHeight).toSeq) - yield CWState (x, y) - - given arbitraryState: Arbitrary[CWState] = Arbitrary (genCWState) - - given eqCWState: Eq[CWState] = Eq.fromUniversalEquals - - given arbitraryReward: Arbitrary[CWReward] = Arbitrary (Gen.double) + /** Here is a proof that our types actually deliver on everything that an Agent + * needs to be able to do to work in the framework. + */ + class CliffWalkingInstances (using probula.RNG) + extends AgentConstraints[CWState, CWObservableState, CWAction, CWReward, Randomized2]: + + given enumAction: BoundedEnumerable[CWAction] = + BoundedEnumerableFromList (CWAction.Up, CWAction.Down, CWAction.Left, CWAction.Right) + + given enumState: BoundedEnumerable[CWObservableState] = + val ss = for + x <- 0 to BoardWidth + y <- 0 to BoardHeight + t <- 0 to TimeHorizon + yield CWState (x, y, t) + BoundedEnumerableFromList (ss*) + + + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad + + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized + + lazy val genCWState: Gen[CWState] = for + x <- Gen.oneOf (0 to BoardWidth) + y <- Gen.oneOf (0 to BoardHeight) + t <- Gen.choose (0, TimeHorizon) + yield CWState (x, y, t) + + given arbitraryState: Arbitrary[CWState] = Arbitrary (genCWState) + + given eqCWState: Eq[CWState] = Eq.fromUniversalEquals + + given arbitraryReward: Arbitrary[CWReward] = Arbitrary (Gen.double) + + given rewardArith: Arith[CWReward] = Arith.arithDouble + + end CliffWalkingInstances - given rewardArith: Arith[CWReward] = Arith.arithDouble +end CliffWalking -end CliffWalkingInstances diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala index a5a44568..296aa24b 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala @@ -18,7 +18,7 @@ class Experiments alpha = 0.1, gamma = 0.1, epsilon0 = 0.1, - episodes = 100 + episodes = 1000 ) s"CliffWalking experiment with $sarsa" in { @@ -27,6 +27,8 @@ class Experiments .take (10) .flatMap { _.headOption } .toList + val fileN = "cliffwalking.csv" + info (s"Starting evaluation and saving to $fileN") eval (sarsa, policies) - .save ("cliffwalking.csv") + .save (fileN) } From 9866cd4428b55cd8a514e117f49d95fb7cc76899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 22:30:06 +0100 Subject: [PATCH 26/32] cliffwalking: Separate observable and external states Otherwise it is difficult to write some tests (becase class members are private for non-case classes apparently) --- src/main/scala/symsim/ExactRL.scala | 3 ++- .../concrete/cliffwalking/CliffWalking.scala | 23 +++++++++++-------- .../concrete/cliffwalking/Experiments.scala | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/scala/symsim/ExactRL.scala b/src/main/scala/symsim/ExactRL.scala index 03d0feec..58be629a 100644 --- a/src/main/scala/symsim/ExactRL.scala +++ b/src/main/scala/symsim/ExactRL.scala @@ -65,7 +65,8 @@ trait ExactRL[State, ObservableState, Action, Reward, Scheduler[_]] def learningEpisode(fR: (VF, List[VF], Probability), s_t: State) : Scheduler[(VF, List[VF], Probability)] = - def done (f: VF, s: State, a: Action): Boolean = agent.isFinal(s) + def done (f: VF, s: State, a: Action): Boolean = + agent.isFinal(s) val (f, qL_t, ε) = fR diff --git a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala index 0ee3f15b..d7c21ae3 100644 --- a/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala +++ b/src/main/scala/symsim/examples/concrete/cliffwalking/CliffWalking.scala @@ -13,7 +13,7 @@ import symsim.concrete.Randomized2 val BoardWidth: Int = 11 val BoardHeight: Int = 3 -class CWObservableState (x: Int, y: Int): +case class CWObservableState (x: Int, y: Int): require (x >= 0, s"Negative horizontal position $x") require (x <= BoardWidth, s"Out-Of-Width x: ¬($x ≤ $BoardWidth)") require (y >= 0, s"Negative vertical board position $y}") @@ -21,8 +21,11 @@ class CWObservableState (x: Int, y: Int): override def toString: String = s"($x,$y)" -case class CWState (x: Int, y: Int, t: Int) - extends CWObservableState (x, y): +case class CWState (x: Int, y: Int, t: Int): + require (x >= 0, s"Negative horizontal position $x") + require (x <= BoardWidth, s"Out-Of-Width x: ¬($x ≤ $BoardWidth)") + require (y >= 0, s"Negative vertical board position $y}") + require (y <= BoardHeight, s"Out-of-Height y: ¬($y ≤ $BoardHeight)") override def toString: String = s"($x,$y,$t)" @@ -38,9 +41,9 @@ enum CWAction: case CWAction.Left => "←" case CWAction.Right => "→" -class CliffWalking (using probula.RNG) - extends Agent[CWState, CWObservableState, CWAction, CWReward, Randomized2] - with Episodic: +class CliffWalking (using probula.RNG) extends + Agent[CWState, CWObservableState, CWAction, CWReward, Randomized2], + Episodic: /** The episode should be guaranteed to terminate after * TimeHorizon steps. This is used *only* *for* testing. It does @@ -52,7 +55,8 @@ class CliffWalking (using probula.RNG) def isFinal (s: CWState): Boolean = (s.y == 0 && s.x > 0) || s.t >= TimeHorizon - def observe (s: CWState): CWObservableState = s + def observe (s: CWState): CWObservableState = + CWObservableState(s.x, s.y) def move (s: CWState, a: CWAction): CWState = a match case CWAction.Up => @@ -78,7 +82,7 @@ class CliffWalking (using probula.RNG) def initialize: Randomized2[CWState] = { for x <- Randomized2.between (0, BoardWidth + 1) y <- Randomized2.between (0, BoardHeight + 1) - s = CWState (x, y, 0) + s = CWState (x, y, 0) yield s }.filter { !this.isFinal (_) } val instances = new CliffWalkingInstances @@ -96,8 +100,7 @@ class CliffWalking (using probula.RNG) val ss = for x <- 0 to BoardWidth y <- 0 to BoardHeight - t <- 0 to TimeHorizon - yield CWState (x, y, t) + yield CWObservableState (x, y) BoundedEnumerableFromList (ss*) diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala index 296aa24b..c739f2b9 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala @@ -23,7 +23,7 @@ class Experiments s"CliffWalking experiment with $sarsa" in { val policies = learnAndLog (sarsa) - .grouped (10) + .grouped (100) .take (10) .flatMap { _.headOption } .toList From b72f4e3e647c0978bd916c9a2ad1522c865e92ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 22:34:03 +0100 Subject: [PATCH 27/32] windygrid: Add timeout to this task to enable early evaluation --- .../concrete/windygrid/WindyGrid.scala | 121 ++++++++++-------- .../concrete/cliffwalking/Experiments.scala | 2 +- .../concrete/windygrid/Experiments.scala | 6 +- .../concrete/windygrid/WindyGridSpec.scala | 10 +- 4 files changed, 75 insertions(+), 64 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala b/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala index 3997a778..228c8e34 100644 --- a/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala +++ b/src/main/scala/symsim/examples/concrete/windygrid/WindyGrid.scala @@ -15,11 +15,14 @@ val xMax = 10 val yMin = 1 val yMax = 7 -case class GridState (x: Int, y: Int): +case class GridObservableState (x: Int, y: Int): require (x >= 1 && x <= 10 && y >= 1 && y <= 7) override def toString: String = s"($x,$y)" -type GridObservableState = GridState +case class GridState (x: Int, y: Int, t: Int): + require (x >= 1 && x <= 10 && y >= 1 && y <= 7) + override def toString: String = s"($x,$y,$t)" + type GridReward = Double enum GridAction: @@ -27,39 +30,42 @@ enum GridAction: class WindyGrid (using probula.RNG) extends - Agent[GridState, GridObservableState, GridAction, GridReward, Randomized2]: + Agent[GridState, GridObservableState, GridAction, GridReward, Randomized2], + Episodic: + + val TimeHorizon: Int = 2000 val WindSpec = Array (0, 0, 0, 1, 1, 1, 2, 2, 1, 0) def isFinal (s: GridState): Boolean = - (s.x, s.y) == (8, 4) + (s.x, s.y) == (8, 4) || s.t >= TimeHorizon - def observe (s: GridState): GridObservableState = - GridState (s.x, s.y) + def observe (s: GridState): GridObservableState = + GridObservableState (s.x, s.y) def gridReward (s: GridState) (a: GridAction): GridReward = s match - case GridState (8, 4) => 0.0 // final state - case GridState (_, _) => -1.0 + case GridState (8, 4, _) => 0.0 // final state + case GridState (_, _, _) => -1.0 def stepRight (s: GridState): GridState = val x1 = (s.x + 1).min (10) val y1 = (s.y+WindSpec(s.x-1)).min (7) - GridState (x1, y1) + GridState (x1, y1, s.t + 1) def stepLeft (s: GridState): GridState = val x1 = (s.x - 1).max (1) val y1 = (s.y + WindSpec (s.x - 1)).min (7) - GridState (x1, y1) + GridState (x1, y1, s.t + 1) def stepUp (s: GridState): GridState = val x1 = s.x val y1 = (s.y + 1 + WindSpec (s.x - 1)).min (7) - GridState (x1, y1) + GridState (x1, y1, s.t + 1) def stepDown (s: GridState): GridState = val x1 = s.x val y1 = ((s.y - 1 + WindSpec (s.x - 1)).max (1)).min (7) - GridState (x1, y1) + GridState (x1, y1, s.t + 1) def step (s: GridState) (a: GridAction): Randomized2[(GridState, GridReward)] = val s1 = a match @@ -69,52 +75,55 @@ class WindyGrid (using probula.RNG) extends case GridAction.D => stepDown (s) Randomized2.const (s1 -> gridReward (s1) (a)) - def initialize: Randomized2[GridState] = { for - x <- Randomized2.between (1, 11) - y <- Randomized2.between (1, 8) - s = GridState (x, y) - yield s }.filter { !this.isFinal (_) } + def initialize: Randomized2[GridState] = { + for x <- Randomized2.between (1, 11) + y <- Randomized2.between (1, 8) + s = GridState (x, y, 0) + yield s + }.filter { !this.isFinal (_) } val instances = new WindyGridInstances -end WindyGrid - -/** Here is a proof that our types actually deliver on everything that an Agent - * needs to be able to do to work in the framework. - */ -class WindyGridInstances (using probula.RNG) - extends AgentConstraints[GridState, GridObservableState, GridAction, - GridReward, Randomized2]: - - given enumAction: BoundedEnumerable[GridAction] = - BoundedEnumerableFromList (U, L, R, D) - - given enumState: BoundedEnumerable[GridObservableState] = - val ss = for - x <- xMin to xMax - y <- yMin to yMax - yield GridState (x, y) - BoundedEnumerableFromList (ss*) - given schedulerIsMonad: Monad[Randomized2] = - Randomized2.randomizedIsMonad + /** Here is a proof that our types actually deliver on everything that an Agent + * needs to be able to do to work in the framework. + */ + class WindyGridInstances (using probula.RNG) + extends AgentConstraints[GridState, GridObservableState, GridAction, + GridReward, Randomized2]: + + given enumAction: BoundedEnumerable[GridAction] = + BoundedEnumerableFromList (U, L, R, D) + + given enumState: BoundedEnumerable[GridObservableState] = + val ss = for + x <- xMin to xMax + y <- yMin to yMax + yield GridObservableState (x, y) + BoundedEnumerableFromList (ss*) + + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad + + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized + + lazy val genGridState: Gen[GridState] = for + x <- Gen.choose (1, 10) + y <- Gen.choose (1, 7) + t <- Gen.choose (0, TimeHorizon) + yield GridState (x, y, t) + + given arbitraryState: Arbitrary[GridState] = + Arbitrary (genGridState) + + given eqGridState: Eq[GridState] = Eq.fromUniversalEquals + + given arbitraryReward: Arbitrary[GridReward] = + Arbitrary (Gen.double) + + given rewardArith: Arith[GridReward] = Arith.arithDouble + + end WindyGridInstances - given canTestInScheduler: CanTestIn[Randomized2] = - Randomized2.canTestInRandomized - - lazy val genGridState: Gen[GridState] = for - x <- Gen.choose (1, 10) - y <- Gen.choose (1, 7) - yield GridState (x, y) - - given arbitraryState: Arbitrary[GridState] = - Arbitrary (genGridState) - - given eqGridState: Eq[GridState] = Eq.fromUniversalEquals - - given arbitraryReward: Arbitrary[GridReward] = - Arbitrary (Gen.double) - - given rewardArith: Arith[GridReward] = Arith.arithDouble - -end WindyGridInstances +end WindyGrid diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala index c739f2b9..fad7e0af 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/Experiments.scala @@ -28,7 +28,7 @@ class Experiments .flatMap { _.headOption } .toList val fileN = "cliffwalking.csv" - info (s"Starting evaluation and saving to $fileN") + info (s"Evaluation report will be written to $fileN") eval (sarsa, policies) .save (fileN) } diff --git a/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala b/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala index a2071218..ae7bb261 100644 --- a/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/windygrid/Experiments.scala @@ -20,10 +20,12 @@ class Experiments s"WindyGrid experiment with $sarsa" in { val policies = learnAndLog (sarsa) - .grouped (10) + .grouped (1000) .take (10) .flatMap { _.headOption } .toList + val fileN = "windygrid.csv" + info (s"Evaluation report will be written to $fileN") val results = eval (sarsa, policies) - results.save ("windygrid.csv") + results.save (fileN) } diff --git a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala index cb651838..a3851781 100644 --- a/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala +++ b/src/test/scala/symsim/examples/concrete/windygrid/WindyGridSpec.scala @@ -21,13 +21,12 @@ import scala.language.adhocExtensions object WindyGridSpec extends org.scalacheck.Properties ("WindyGrid"): - import windyGrid.instances.{arbitraryAction, arbitraryState, enumState, enumAction} + import windyGrid.instances.given property ("Up and Down will never affect the x value") = forAll { (s: GridState) => - for - (s1, r) <- windyGrid.step (s) (GridAction.U) - (s2, r) <- windyGrid.step (s) (GridAction.D) + for (s1, r) <- windyGrid.step (s) (GridAction.U) + (s2, r) <- windyGrid.step (s) (GridAction.D) yield s1._1 == s.x && s2._1 == s.x } @@ -48,10 +47,11 @@ object WindyGridSpec // to rerun training multiple times (if it is put inside the property) val sarsa = ConcreteSarsa (windyGrid, 0.1, 0.5, 0.1, 1) val initials = LazyList (windyGrid.instances.allObservableStates*) + .map { (s: GridObservableState) => GridState(s.x, s.y, 0) } val qs = sarsa.learn (sarsa.vf.initialize, List[sarsa.vf.Q] (), initials) .sample (5) import sarsa.vf.apply property ("Q-table values are non-positive") = - forAll { (s: GridState, a: GridAction) => + forAll { (s: GridObservableState, a: GridAction) => qs.forall { (q, _) => q (s, a) <= 0 } } From dd10e24c80f811a78ac658188affb472d16ad7a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 22:43:11 +0100 Subject: [PATCH 28/32] mountaincar: Add time horizon to enable evaluation --- .../concrete/mountaincar/MountainCar.scala | 102 +++++++++--------- .../concrete/mountaincar/Experiments.scala | 6 +- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala b/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala index aacb3ade..586b56cb 100644 --- a/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala +++ b/src/main/scala/symsim/examples/concrete/mountaincar/MountainCar.scala @@ -22,10 +22,12 @@ import symsim.concrete.Randomized2.given * the same type to represent the finite state space. */ -case class CarState (v: Double, p: Double): +case class CarState (v: Double, p: Double, t: Int): + override def toString: String = f"[v $v%+2.2f, p $p%+2.2f, t $t]" + +case class CarObservableState (v: Double, p: Double): override def toString: String = f"[v $v%+2.2f, p $p%+2.2f]" -type CarObservableState = CarState type CarAction = Double type CarReward = Double @@ -34,7 +36,7 @@ class MountainCar (using RNG) extends Agent[CarState, CarObservableState, CarAction, CarReward, Randomized2] with Episodic: - val TimeHorizon: Int = 3000000 + val TimeHorizon: Int = 2000 def roundAt (p: Int) (n: Double): Double = val s = Math.pow (10, p) @@ -42,7 +44,7 @@ class MountainCar (using RNG) def isFinal (s: CarState): Boolean = - s.p >= 0.5 + s.p >= 0.5 || s.t >= TimeHorizon def observe (s: CarState): CarObservableState = @@ -53,7 +55,7 @@ class MountainCar (using RNG) val dp = roundAt (2) (-1.2 + (((s.p + 1.2) / 0.17).floor) * 0.17) val dv = roundAt (2) (-1.5 + (((s.v + 1.5) / 0.30).floor) * 0.30) - CarState (v = dv.min (1.5).max (-1.5), p = dp.min (0.5).max (-1.2)) + CarObservableState (v = dv.min (1.5).max (-1.5), p = dp.min (0.5).max (-1.2)) private def carReward (s: CarState) (a: CarAction): CarReward = @@ -73,60 +75,64 @@ class MountainCar (using RNG) val p = s.p + (v * dt) val (v1, p1) = if p < -1.2 then (-1.2, 0.0) else (v, p) val s1 = CarState (v = v1.max (-1.5).min (1.5), - p = p1.min (0.5)) + p = p1.min (0.5), + t = s.t + 1) Randomized2.const (s1 -> carReward (s1) (a)) def initialize: Randomized2[CarState] = (for p <- Randomized2.between (-1.2, 0.5) v <- Randomized2.between (-1.5, 1.5) - s = CarState (v=v, p=p) + s = CarState (v=v, p=p, 0) yield s).filter { !this.isFinal (_) } val instances = new MountainCarInstances -end MountainCar - - -/** Here is a proof that our types actually deliver on everything that an Agent - * needs to be able to do to work in the framework. - */ -class MountainCarInstances (using RNG) - extends AgentConstraints[CarState, CarObservableState, CarAction, CarReward, Randomized2]: - - given enumAction: BoundedEnumerable[CarAction] = - BoundedEnumerableFromList (-0.2, 0.0, 0.2) - - given enumState: BoundedEnumerable[CarObservableState] = - val ss = for - p0 <- Seq (-1.2, -1.03, -0.86, -0.69, -0.52, -0.35, -0.18, -0.01, 0.16, 0.33, 0.5) - v0 <- Seq (-1.5, -1.2, -0.9, -0.6, -0.3, 0.0, 0.3, 0.6, 0.9, 1.2, 1.5) - yield CarState (v = v0, p = p0) - BoundedEnumerableFromList (ss*) - - given schedulerIsMonad: Monad[Randomized2] = - Randomized2.randomizedIsMonad - - given canTestInScheduler: CanTestIn[Randomized2] = - Randomized2.canTestInRandomized - - lazy val genCarState: Gen[CarState] = for - p <- Gen.choose (-1.2, 0.5) - v <- Gen.choose (-1.5, 1.5) - yield CarState (v, p) - - given arbitraryState: Arbitrary[CarState] = Arbitrary (genCarState) - - given eqCarState: Eq[CarState] = Eq.fromUniversalEquals - - /** This is useful to limit as it is used in tests and - * initialization of Q tables. If these values are unreasonably - * large they will break statistical tests. + /** Here is a proof that our types actually deliver on everything that an Agent + * needs to be able to do to work in the framework. */ - given arbitraryReward: Arbitrary[CarReward] = - Arbitrary (Gen.choose (-300.0, 300.0)) + class MountainCarInstances (using RNG) + extends AgentConstraints[CarState, CarObservableState, CarAction, CarReward, Randomized2]: + + given enumAction: BoundedEnumerable[CarAction] = + BoundedEnumerableFromList (-0.2, 0.0, 0.2) + + given enumState: BoundedEnumerable[CarObservableState] = + val ss = for + p0 <- Seq (-1.2, -1.03, -0.86, -0.69, -0.52, -0.35, -0.18, -0.01, 0.16, 0.33, 0.5) + v0 <- Seq (-1.5, -1.2, -0.9, -0.6, -0.3, 0.0, 0.3, 0.6, 0.9, 1.2, 1.5) + yield CarObservableState (v = v0, p = p0) + BoundedEnumerableFromList (ss*) + + + given schedulerIsMonad: Monad[Randomized2] = + Randomized2.randomizedIsMonad + + given canTestInScheduler: CanTestIn[Randomized2] = + Randomized2.canTestInRandomized + + lazy val genCarState: Gen[CarState] = for + p <- Gen.choose (-1.2, 0.5) + v <- Gen.choose (-1.5, 1.5) + t <- Gen.choose (0, TimeHorizon) + yield CarState (v = v, p = p, t = t) + + given arbitraryState: Arbitrary[CarState] = Arbitrary (genCarState) + + given eqCarState: Eq[CarState] = Eq.fromUniversalEquals + + /** This is useful to limit as it is used in tests and + * initialization of Q tables. If these values are unreasonably + * large they will break statistical tests. + */ + given arbitraryReward: Arbitrary[CarReward] = + Arbitrary (Gen.choose (-300.0, 300.0)) + + given rewardArith: Arith[CarReward] = Arith.arithDouble + + end MountainCarInstances + - given rewardArith: Arith[CarReward] = Arith.arithDouble +end MountainCar -end MountainCarInstances diff --git a/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala b/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala index eea19962..466bacb5 100644 --- a/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala +++ b/src/test/scala/symsim/examples/concrete/mountaincar/Experiments.scala @@ -28,6 +28,8 @@ class Experiments .take (10) .flatMap { _.headOption } .toList - val results = eval (sarsa, policies) - results.save ("mountaincar.csv") + val fileN = "mountaincar.csv" + info (s"Evaluation report will be written to $fileN") + eval (sarsa, policies) + .save (fileN) } From 506ccf31624197b6b5db9556473d66cfebb61a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 23:06:59 +0100 Subject: [PATCH 29/32] probula: Allow using different Generators than Secure Random from Java --- src/main/scala/probula/Dist.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/probula/Dist.scala b/src/main/scala/probula/Dist.scala index d9e9b2e6..53a57208 100644 --- a/src/main/scala/probula/Dist.scala +++ b/src/main/scala/probula/Dist.scala @@ -27,7 +27,7 @@ import spire.random.Uniform.* import spire.random.Dist.* import spire.random.* -type RNG = spire.random.rng.SecureJava +type RNG = spire.random.Generator /** A representation of probabilistic models as multivariate * distributions, effectively hierarchical Bayesian models. From 51d26deb2b61a6777cf1de5b86efebd799f63d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 23:07:14 +0100 Subject: [PATCH 30/32] randomized2: Migrate the randomized spec to Randomized2 One test fails, but this is the same bug in spire as others --- .../symsim/concrete/RandomizedSpec.scala | 116 +++++++----------- 1 file changed, 46 insertions(+), 70 deletions(-) diff --git a/src/test/scala/symsim/concrete/RandomizedSpec.scala b/src/test/scala/symsim/concrete/RandomizedSpec.scala index 31cd5bf6..516463f9 100644 --- a/src/test/scala/symsim/concrete/RandomizedSpec.scala +++ b/src/test/scala/symsim/concrete/RandomizedSpec.scala @@ -2,115 +2,91 @@ package symsim package concrete import scala.util.Try +import cats.syntax.all.* import org.scalacheck.Prop.{forAll, forAllNoShrink, propBoolean} import org.scalacheck.Gen -import symsim.concrete.Randomized.canTestInRandomized +import symsim.concrete.Randomized2.canTestInRandomized import symsim.CanTestIn.* -/** Sanity tests for Randomized as a Scheduler */ -class RandomizedSpec - extends org.scalatest.freespec.AnyFreeSpec, - org.scalatestplus.scalacheck.Checkers: +/** Sanity tests for Randomized2 as a Scheduler */ +class RandomizedSpec extends + org.scalatest.freespec.AnyFreeSpec, + org.scalatestplus.scalacheck.Checkers: given PropertyCheckConfiguration = PropertyCheckConfiguration(minSuccessful = 100) val C = 5000 - "Sanity checks for symsim.concrete.Randomized" - { - - "between Double observes the bounds" in check { - forAll { (mn: (Double, Double)) => - val m = Math.min (mn._1, mn._2) - val n = Math.max (mn._1, mn._2) - val between = Randomized.between (m, n) - m != n ==> - forAll (between.toGen) { x => m <= x && x < n } - } - } - - "between Int observes the bounds" in check { - forAll { (mn: (Int, Int)) => - val m = Math.min (mn._1, mn._2) - val n = Math.max (mn._1, mn._2) - m != n ==> - forAll (Randomized.between (m, n).toGen) { x => m <= x && x < n } - } - } - } + // "Sanity checks for symsim.concrete.Randomized2" - { + + // "between Double observes the bounds" in check { + // forAll { (mn: (Double, Double)) => + // val m = Math.min (mn._1, mn._2) + // val n = Math.max (mn._1, mn._2) + // val between = Randomized2.between (m, n) + // m != n ==> + // forAll (between.toGen) { x => m <= x && x < n } + // } + // } + + // "between Int observes the bounds" in check { + // forAll { (mn: (Int, Int)) => + // val m = Math.min (mn._1, mn._2) + // val n = Math.max (mn._1, mn._2) + // m != n ==> + // forAll (Randomized2.between (m, n).toGen) { x => m <= x && x < n } + // } + // } + // } "Regressions" - { - def isSingleton[A] (ra: Randomized[A]): Boolean = - Try { !ra.isEmpty && ra.tail.isEmpty }.get - - "Randomized.const is finite (a regression)" in { - assert { isSingleton (Randomized.const(1)) } - } - - "Randomized.between(Int, Int) is finite (a regression)" in { - assert { isSingleton (Randomized.between(1, 42)) } - } - - "Randomized.between(Double, Double) is finite (a regression)" in { - assert { isSingleton (Randomized.between(-1.0, 2.0)) } - } - - "Randomized.coin(Double) is finite (a regression)" in { - assert { isSingleton (Randomized.coin(0.5)) } - } - - "Randomized.oneOf(Int*) is finite (a regression)" in { - assert { isSingleton (Randomized.oneOf(1,2,3,5)) } - } - - def repeatNotConstant[T: Numeric] (ra: =>Randomized[T]): Boolean = - Try { Randomized.repeat (ra) - .take (20) - .toList - .distinct - .size > 1 }.get + def repeatNotConstant[T: Numeric] (ra: =>Randomized2[T]): Boolean = + Try { ra.sample (20) + .toList + .distinct + .size > 1 }.get - def repeatNotConstantB (ra: => Randomized[Boolean]): Boolean = + def repeatNotConstantB (ra: => Randomized2[Boolean]): Boolean = repeatNotConstant[Int] (ra map { x => if x then 1 else 0 }) "20 consecutive values of const(42)* are not different" in { - assert { !repeatNotConstant (Randomized.const (42)) } + assert { !repeatNotConstant (Randomized2.const (42)) } } "20 consecutive values of between(1,100)* are random (different)" in { - assert { repeatNotConstant (Randomized.between (1,100)) } + assert { repeatNotConstant (Randomized2.between (1,100)) } } "20 consecutive values of between(-1.0,1.0)* are random (different)" in { - assert { repeatNotConstant (Randomized.between (1.0, 100.0)) } + assert { repeatNotConstant (Randomized2.between (1.0, 100.0)) } } "20 consecutive values of coin(.5)* are random (different)" in { - assert { repeatNotConstantB (Randomized.coin (.5)) } + assert { repeatNotConstantB (Randomized2.coin (.5)) } } "20 consecutive values of coin(.1)* are not different" in { - assert { !repeatNotConstantB (Randomized.coin (1.0)) } + assert { !repeatNotConstantB (Randomized2.coin (1.0)) } } "20 consecutive values of oneOf(1..100)* are random (different)" in { - assert { repeatNotConstant (Randomized.oneOf (1 to 100*)) } + assert { repeatNotConstant (Randomized2.oneOf (1 to 100*)) } } /* This test records what is the problem with referential transparency in - * the current implementation of Randomized. + * the current implementation of Randomized2. */ - "Randomized is referentially transparent (failing)" ignore { - assert ( - Randomized.repeat (Randomized.between(1,100)).take (10).toList == - Randomized.repeat (Randomized.between(1,100)).take (10).toList - ) + "Randomized2 is referentially transparent" in check { + val rng = spire.random.rng.Serial (42) + Randomized2.between(1,100).sample (10) (using rng).toList == + Randomized2.between(1,100).sample (10) (using rng).toList } - "Randomized.gaussian (m, d) has stddev 'd' and mean 'm'" in check { + "Randomized2.gaussian (m, d) has stddev 'd' and mean 'm'" in check { val n = 20000 val epsilon = 0.06 @@ -118,7 +94,7 @@ class RandomizedSpec forAllNoShrink (Gen.choose (-100.0, +100.0), Gen.choose(0.1, 2.0)) { (m: Double, d: Double) => Math.abs (m) >= 0.5 ==> { - val sample = Randomized.repeat (Randomized.gaussian (mean = m, stddev = d)).take (n) + val sample = Randomized2.gaussian (mean = m, stdDev = d).sample (n) val mean = sample.sum / n val variance = sample.map { x => (x-mean)*(x-mean) }.sum / n val mm = Math.abs ((mean-m)/ m) From 79f2a65cea4905b290c8025814bcc7a6cb36c6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Sat, 13 Jan 2024 23:11:41 +0100 Subject: [PATCH 31/32] Randomized: Remove the old randomized from the source --- src/main/scala/symsim/QTable.scala | 2 +- .../scala/symsim/concrete/Randomized.scala | 150 ------------------ .../scala/symsim/concrete/Randomized2.scala | 2 + src/main/scala/symsim/laws/EpisodicLaws.scala | 1 - .../examples/concrete/braking/CarSpec.scala | 1 - .../cliffwalking/CliffWalkingSpec.scala | 1 - .../concrete/simplemaze/MazeSpec.scala | 1 - 7 files changed, 3 insertions(+), 155 deletions(-) delete mode 100644 src/main/scala/symsim/concrete/Randomized.scala diff --git a/src/main/scala/symsim/QTable.scala b/src/main/scala/symsim/QTable.scala index a1e9fce4..4e44d050 100644 --- a/src/main/scala/symsim/QTable.scala +++ b/src/main/scala/symsim/QTable.scala @@ -10,7 +10,7 @@ import org.scalacheck.Gen import org.scalacheck.Arbitrary import org.scalacheck.Arbitrary.arbitrary import org.typelevel.paiges.Doc -import symsim.concrete.{ConcreteExactRL, Randomized} +import symsim.concrete.ConcreteExactRL import symsim.Arith.* diff --git a/src/main/scala/symsim/concrete/Randomized.scala b/src/main/scala/symsim/concrete/Randomized.scala deleted file mode 100644 index 71fbfe42..00000000 --- a/src/main/scala/symsim/concrete/Randomized.scala +++ /dev/null @@ -1,150 +0,0 @@ -package symsim.concrete - -import scala.annotation.targetName - -import org.scalacheck.{Gen, Prop} - -import scala.jdk.StreamConverters.* -import scala.util.Try - -import java.security.SecureRandom - -/** A purely functional wrapping of java.security.SecureRandom (not really pure, - * but hides things sufficiently for now). - */ -type Randomized[+A] = LazyList[A] - -type Probability = Double - - -/** A purely functional wrapping of java.security.SecureRandom. Presents - * generators as possibly finite streams (a source that dries out), which can - * be extended to continue producing infinitely with repeat (infinite). There - * is some loss of referential transparency in this switch to infinity - * (repeat). - * - * Randomized should be a non-branching scheduler. For the infrastructure to - * work, all our schedulers need to be finitely branching. - **/ -object Randomized: - - /** Create a generator that produces a single 'a'. Used to create - * deterministic values when a scheduler/randomized type is - * expected. - */ - def const[A] (a: =>A): Randomized[A] = - LazyList (a) - - def prob: Randomized[Probability] = - LazyList (SecureRandom ().nextDouble) - - - /** Produce a single random number between the bounds, - * right exclusive - **/ - def between (minInclusive: Int, maxExclusive: Int): Randomized[Int] = - LazyList (SecureRandom () - .ints (minInclusive, maxExclusive) - .findAny - .getAsInt) - - - def between (minInclusive: Double, maxExclusive: Double): Randomized[Double] = - LazyList (SecureRandom () - .doubles (minInclusive, maxExclusive) - .findAny - .getAsDouble) - - def gaussian (mean: Double = 0.0, stddev: Double = 1.0): Randomized[Double] = - LazyList (SecureRandom ().nextGaussian * stddev + mean) - - /** Toss a coing biased towards true with probabilty 'bias' */ - def coin (bias: Probability): Randomized[Boolean] = - LazyList (SecureRandom ().nextDouble <= bias) - - def oneOf[A] (choices: A*): Randomized[A] = - between (0, choices.size) - .map { i => choices (i) } - - def eachOf[A] (choices: A*): Randomized[A] = - LazyList(choices*) - - def repeat[A] (ra: =>Randomized[A]): Randomized[A] = - LazyList.continually (ra).flatten - - given randomizedIsMonad: cats.Monad[Randomized] = - cats.instances.lazyList.catsStdInstancesForLazyList - - given canTestInRandomized: symsim.CanTestIn[Randomized] = - new symsim.CanTestIn[Randomized] { - - @targetName ("toPropBoolean") - def toProp (rProp: Randomized[Boolean]) = - Prop.forAllNoShrink (toGen (rProp)) (identity[Boolean]) - - // This is a nasty hack that costs as a lot on memory in tests (but - // probably not in experiments). Unfortunately, I do not see an easy way - // to add a completely new generator for scalacheck that encapsulates - // Randomized. The Gen class appears to be sealed and pimping cannot add - // state to objects? - def toGen[A] (ra: => Randomized[A]): Gen[A] = - require (ra.nonEmpty) - val stream = repeat (ra) - Gen.choose(0, 1000) - .map { i => stream (i) } - } - - /** This extensions should ideally be used at a top-level of the program, - * as they loose the type annotation for the side effect of randomness. - */ - extension [A] (self: Randomized[A]) - /** Get one sample from randomized. Note that the sample will be random but - * always the same if you call several times. - * (at least in the current implementation) - */ - def sample (): A = sample(1).head - - /** Get n samples from randomized. Note that the sample will be random but - * always the same if you call several times. - * (at least in the current implementation) - * - * Note: as long as we have not refactored the underlying implementation - * of Randomized, this is unsafe. It is possible that randomized does - * not have n samples. - * - */ - def sample (n: Int): LazyList[A] = self.take(n) - - /** Perform an imperative operation that depends on one sample from this - * Randomized. This is mostly meant for IO at this point. - */ - def run (f: A => Unit): Unit = f(self.sample ()) - - - extension (self: Randomized[Double]) - - /** Calculate a mean of this random variable. - * Note that calling mean might be very expensive, if obtaining each sample - * is expensive. - */ - def mean (support: Int = 100): Double = - val finite = self.take (support) - finite.sum / support.toDouble - - def variance (support: Int = 100): Double = - val finite = self.take (support) - val μ = finite.mean (support) - self.map (x => (x - μ)*(x -μ)) - .mean (support) - - /** Calculate a mean and variance of a sample of size support. This is done - * in one go, so it is more efficient than calling mean and variance - * separately. - */ - def meanVar (support: Int = 100): (Double, Double) = - val finite = self.take (support) - val μ = finite.mean (support) - val σσ = self.map (x => (x - μ)*(x -μ)) - .mean (support) - (μ, σσ) - diff --git a/src/main/scala/symsim/concrete/Randomized2.scala b/src/main/scala/symsim/concrete/Randomized2.scala index 9bca9d3d..0d445fde 100644 --- a/src/main/scala/symsim/concrete/Randomized2.scala +++ b/src/main/scala/symsim/concrete/Randomized2.scala @@ -6,6 +6,8 @@ import probula.{Dist, Name, IData} opaque type Randomized2[+A] = Dist[A] +type Probability = Double + object Randomized2: /** Create a generator that produces a single 'a'. Used to create diff --git a/src/main/scala/symsim/laws/EpisodicLaws.scala b/src/main/scala/symsim/laws/EpisodicLaws.scala index 7436ebe7..863bb9e1 100644 --- a/src/main/scala/symsim/laws/EpisodicLaws.scala +++ b/src/main/scala/symsim/laws/EpisodicLaws.scala @@ -13,7 +13,6 @@ import org.scalacheck.Prop import org.scalacheck.Prop.forAll import org.scalacheck.Prop.forAllNoShrink -import symsim.concrete.Randomized import symsim.CanTestIn.* /** Collect the laws that define correctness of an Agent with Episodic. diff --git a/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala b/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala index 72fc9759..37a40ca4 100644 --- a/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala +++ b/src/test/scala/symsim/examples/concrete/braking/CarSpec.scala @@ -7,7 +7,6 @@ import org.scalacheck.Gen import org.scalacheck.Prop.* import symsim.CanTestIn.given -import symsim.concrete.Randomized.given import car.instances.{arbitraryState, arbitraryAction} diff --git a/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala b/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala index 12a19562..c7ac0304 100644 --- a/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala +++ b/src/test/scala/symsim/examples/concrete/cliffwalking/CliffWalkingSpec.scala @@ -5,7 +5,6 @@ import org.scalacheck.Gen import org.scalacheck.Prop.forAll import symsim.CanTestIn.given -import symsim.concrete.Randomized.given private val cliffWalking = new CliffWalking (using spire.random.rng.SecureJava.apply) diff --git a/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala b/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala index 4d011ad2..1d240635 100644 --- a/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala +++ b/src/test/scala/symsim/examples/concrete/simplemaze/MazeSpec.scala @@ -6,7 +6,6 @@ import org.scalacheck.Gen import org.scalacheck.Prop.forAll import symsim.CanTestIn.given -import symsim.concrete.Randomized.given private val maze = new Maze (using spire.random.rng.SecureJava.apply) import maze.instances.{arbitraryAction, arbitraryState} From 31975af103843f94a40e4e37c78f13af37c32ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20W=C4=85sowski?= Date: Tue, 5 Mar 2024 12:00:13 +0100 Subject: [PATCH 32/32] probula: Improve several comments + ws changes --- src/main/scala/symsim/ExactRL.scala | 13 +++++++++++-- .../scala/symsim/concrete/ConcreteExactRL.scala | 6 ++++++ src/main/scala/symsim/concrete/ConcreteQTable.scala | 2 -- src/main/scala/symsim/concrete/ConcreteVTable.scala | 4 ++-- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/scala/symsim/ExactRL.scala b/src/main/scala/symsim/ExactRL.scala index 58be629a..6dc1c8ee 100644 --- a/src/main/scala/symsim/ExactRL.scala +++ b/src/main/scala/symsim/ExactRL.scala @@ -79,8 +79,17 @@ trait ExactRL[State, ObservableState, Action, Reward, Scheduler[_]] /** Executes as many full learning episodes (until the final state of agent * is reached) as the given state LazyList generates. The list should be - * finite. We force the evaluation when we are done to return the value. - */ + * finite. Learn is lazy, it just constructs a sampler that is able to sample + * outcomes of learning. Each sample costs executing an episode. We force + * the evaluation when we are done to return the value (in runQ). + * + * @param f the initial value function (for instance a Q-table) + * @param q_l the initial (empty) list of past value functions (a log) + * @param ss a lazy list of initial states for the episode. Should be finite. + * + * @return a scheduler (a sampler) of final Q-Tables and intermediate + * Q-tables that led to them + */ final def learn (f: VF, q_l: List[VF], ss: LazyList[State]) : Scheduler[(VF, List[VF])] = val result = diff --git a/src/main/scala/symsim/concrete/ConcreteExactRL.scala b/src/main/scala/symsim/concrete/ConcreteExactRL.scala index 5ff51fa2..d88eee9a 100644 --- a/src/main/scala/symsim/concrete/ConcreteExactRL.scala +++ b/src/main/scala/symsim/concrete/ConcreteExactRL.scala @@ -24,6 +24,12 @@ trait ConcreteExactRL[State, ObservableState, Action] given rng: probula.RNG + /** Execute the learning process for this.episodes number of episodes. + * It initializes the process and calls learn to construct a sampler + * of outcomes + * + * @return A pair: the final Q-Table and a list of intermediate Q-Tables. + */ def runQ: (Q, List[Q]) = val initials = agent.initialize.sample (episodes) val outcome = learn (vf.initialize, List[VF] (), initials).sample () diff --git a/src/main/scala/symsim/concrete/ConcreteQTable.scala b/src/main/scala/symsim/concrete/ConcreteQTable.scala index 410c7b75..109a86f8 100644 --- a/src/main/scala/symsim/concrete/ConcreteQTable.scala +++ b/src/main/scala/symsim/concrete/ConcreteQTable.scala @@ -9,8 +9,6 @@ import symsim.QTable import symsim.concrete.Randomized2.given -// import cats.Monad.nonInheritedOps.toFlatMapOps - class ConcreteQTable[ObservableState: BoundedEnumerable, Action: BoundedEnumerable] extends QTable[ObservableState, Action, Double, Randomized2]: diff --git a/src/main/scala/symsim/concrete/ConcreteVTable.scala b/src/main/scala/symsim/concrete/ConcreteVTable.scala index d4428b44..1931095f 100644 --- a/src/main/scala/symsim/concrete/ConcreteVTable.scala +++ b/src/main/scala/symsim/concrete/ConcreteVTable.scala @@ -19,6 +19,6 @@ trait ConcreteVTable[State, ObservableState, Action] def chooseAction (v: V) (s: State): Scheduler[Action] = for explore <- Randomized2.coin (this.epsilon0) action <- if explore - then Randomized2.oneOf (allActions*) - else Randomized2.const (bestAction (v) (s)) + then Randomized2.oneOf (allActions*) + else Randomized2.const (bestAction (v) (s)) yield action