Add optimizers to FTorch (#320)

* Optimizer functionality added to FTorch.
* A new module ftorch_optim added which contains implementations of 
  SGD, Adam, and AdamW as well as core optimizer methods. Made available
  at top level ftorch.
* Matches as closely as possible the behaviour from PyTorch.
* A new worked example training a single tensor as part of a loop is included.
* Comprehensive unit tests for each optimizer and core functionalities
* Detailed documentation for Optimizer API and online training added.

Author: jatkinson1000

.github/workflows/static_analysis.yml

@@ -110,6 +110,7 @@ jobs:
           fortitude check src/ftorch_types.f90
           fortitude check src/ftorch_tensor.f90
           fortitude check src/ftorch_model.f90
+          fortitude check src/ftorch_optim.f90
           fortitude check src/ftorch_test_utils.f90
 
       # Apply C++ and C linter and formatter, clang

.github/workflows/test_suite_macos_cpu_clang.yml

@@ -224,7 +224,7 @@ jobs:
           export Torch_DIR="${VIRTUAL_ENV}/lib/python${VN}/site-packages/torch"
           export FTORCH_INSTALL_DIR=/tmp/ftorch-install
           export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:${VIRTUAL_ENV}/lib/python${VN}/site-packages/torch/lib:${FTORCH_BUILD_DIR}/lib
-          export EXAMPLE_BUILD_DIR="examples/2_SimpleNet/build"
+          export EXAMPLE_BUILD_DIR="examples/02_SimpleNet/build"
           mkdir "${EXAMPLE_BUILD_DIR}"
           cd "${EXAMPLE_BUILD_DIR}"
           cmake .. \

.github/workflows/test_suite_ubuntu_cpu_gnu.yml

@@ -230,7 +230,7 @@ jobs:
           VN=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))")
           export Torch_DIR="${VIRTUAL_ENV}/lib/python${VN}/site-packages/torch"
           export FTORCH_INSTALL_DIR="/tmp/ftorch-install"
-          export EXAMPLE_BUILD_DIR="examples/2_SimpleNet/build"
+          export EXAMPLE_BUILD_DIR="examples/02_SimpleNet/build"
           mkdir "${EXAMPLE_BUILD_DIR}"
           cd "${EXAMPLE_BUILD_DIR}"
           cmake .. \

.github/workflows/test_suite_ubuntu_cpu_intel.yml

@@ -145,7 +145,7 @@ jobs:
           VN=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))")
           export Torch_DIR="${VIRTUAL_ENV}/lib/python${VN}/site-packages/torch"
           export FTORCH_INSTALL_DIR="/tmp/ftorch-install"
-          export EXAMPLE_BUILD_DIR="examples/2_SimpleNet/build"
+          export EXAMPLE_BUILD_DIR="examples/02_SimpleNet/build"
           mkdir "${EXAMPLE_BUILD_DIR}"
           cd "${EXAMPLE_BUILD_DIR}"
           cmake .. \

.github/workflows/test_suite_windows_cpu_intel.yml

@@ -116,7 +116,7 @@ jobs:
           set FTORCH_INSTALL_DIR=%TEMP%\ftorch-install
           set PATH=%FTORCH_INSTALL_DIR%\bin;%PATH%
           set PATH=%TORCH_PATH%\torch\lib;%PATH%
-          set EXAMPLE_BUILD_DIR=examples\2_SimpleNet\build
+          set EXAMPLE_BUILD_DIR=examples\02_SimpleNet\build
           if not exist %EXAMPLE_BUILD_DIR% mkdir %EXAMPLE_BUILD_DIR%
           cd %EXAMPLE_BUILD_DIR%
           cmake .. ^

.gitignore

@@ -210,3 +210,6 @@ CMakeUserPresets.json
 
 # Apple Desktop Services Store
 **.DS_Store
+
+# Example outputs
+*.png

CHANGELOG.md

@@ -113,7 +113,8 @@ endif()
 # Library with C and Fortran bindings
 add_library(${LIB_NAME} src/ctorch.cpp src/ftorch.f90 src/ftorch_devices.F90
                         src/ftorch_types.f90 src/ftorch_tensor.f90
-                        src/ftorch_model.f90 src/ftorch_test_utils.f90)
+                        src/ftorch_model.f90 src/ftorch_optim.f90
+                        src/ftorch_test_utils.f90)
 
 # Define compile definitions, including GPU devices
 target_compile_definitions(
@@ -187,6 +188,8 @@ install(FILES "${CMAKE_Fortran_MODULE_DIRECTORY}/ftorch_tensor.mod"
         DESTINATION "${CMAKE_INSTALL_MODULEDIR}")
 install(FILES "${CMAKE_Fortran_MODULE_DIRECTORY}/ftorch_model.mod"
         DESTINATION "${CMAKE_INSTALL_MODULEDIR}")
+install(FILES "${CMAKE_Fortran_MODULE_DIRECTORY}/ftorch_optim.mod"
+        DESTINATION "${CMAKE_INSTALL_MODULEDIR}")
 install(FILES "${CMAKE_Fortran_MODULE_DIRECTORY}/ftorch_test_utils.mod"
         DESTINATION "${CMAKE_INSTALL_MODULEDIR}")