!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!
module cubeadm_taskloop
  use gkernel_interfaces
  use cubetools_parameters
  use cubetuple_format
  use cubedag_link_type
  use cubetopology_cuberegion_types
  use cube_types
  use cubeadm_messaging
  use cubeadm_entryloop
  use cubeadm_taskloop_iteration
  !
  integer(kind=code_k), parameter :: code_1entry_for_allplanes=-1
  !
  type :: cubeadm_iterator_cubedesc_t
    ! Convenient type which gathers temporarily all the cubes relevant
    ! information needed to compute the iterator.
    logical                          :: input   ! Is this an input or output cube?
    class(format_t),         pointer :: cube    ! The associated cube
    type(cuberegion_prog_t), pointer :: region  ! The associated region, if relevant
    integer(kind=code_k)             :: access  ! Iterator access mode
  end type cubeadm_iterator_cubedesc_t
  !
  type :: cubeadm_iterator_t
    ! Entries
    integer(kind=entr_k), public  :: ne          ! Total number of entries
    integer(kind=entr_k), private :: neperplane  ! Number of output entries per plane
    ! Tasks
    integer(kind=entr_k), public  :: nt          ! Total number of tasks
    ! Planes
    integer(kind=entr_k), private :: np          ! Total number of reference planes
    integer(kind=entr_k), private :: nppertask   ! Number of planes per task
    ! Current task iteration
    ! NB: this is a duplicate of taskloop_iteration_t. Ifort 13 and 18 have
    !     a bug which does not correctly duplicates sub-types components in
    !     FIRSTPRIVATE statements. No better way than duplicating the type
    !     here
    integer(kind=entr_k), public  :: num         ! This task number
    integer(kind=entr_k), public  :: ie          ! The output entry number iterated in this task
    integer(kind=entr_k), private :: ofe,ole     ! Output first/last entry to be processed by this task
    integer(kind=entr_k), public  :: prange(2)   ! Ref    first/last plane to be processed by this task
  contains
    procedure, public :: iterate_entry => cubeadm_iterator_iterate_entry
  end type cubeadm_iterator_t
  !
  public :: cubeadm_iterator_t
  public :: cubeadm_taskloop_init,cubeadm_taskloop_iterate
  private
  !
contains
  !
  subroutine cubeadm_taskloop_init(incubes,oucubes,align,region,iter,error)
    use cubedag_node_type
    !----------------------------------------------------------------------
    ! Set up the iterator suited for the input and the output cubes
    !----------------------------------------------------------------------
    type(cubedag_link_t),     intent(in)    :: incubes  ! In cube list
    type(cubedag_link_t),     intent(in)    :: oucubes  ! Out cube list
    integer(kind=code_k),     intent(in)    :: align    !
    type(cuberegion_prog_t),  pointer       :: region   !
    type(cubeadm_iterator_t), intent(out)   :: iter
    logical,                  intent(inout) :: error
    ! Local
    character(len=*), parameter :: rname='TASKLOOP>INIT'
    type(cubeadm_iterator_cubedesc_t), allocatable :: myiter(:)
    integer(kind=entr_k) :: icube,ncube
    class(format_t), pointer :: cube
    integer(kind=4) :: ier
    class(cubedag_node_object_t), pointer :: dno
    character(len=message_length) :: mess
    !
    call cubeadm_message(admseve%trace,rname,'Welcome')
    !
    ! Compute one iterator per cube
    allocate(myiter(incubes%n+oucubes%n),stat=ier)
    if (failed_allocate(rname,'iterator',ier,error)) return
    ncube = 0
    !
    ! Input cubes
    do icube=1,incubes%n
      dno => cubedag_node_ptr(incubes%list(icube)%p,error)
      if (error) return
      cube => cubetuple_format_ptr(dno,error)
      if (error) return
      ! Skip cubes for which we won't iterate data
      if (cube%tuple%current%desc%action.eq.code_read_head)  cycle
      ! Set up an iterator suited for this cube
      ncube = ncube+1
      write(mess,'(A,I0)') '--- Computing iterator for cube #',ncube
      call cubeadm_message(admseve%others,rname,mess)
      call cubeadm_taskloop_cubedesc_fill(cube,.true.,region,myiter(ncube),error)
      if (error) return
    enddo
    !
    ! Output cubes
    do icube=1,oucubes%n
      dno => cubedag_node_ptr(oucubes%list(icube)%p,error)
      if (error) return
      cube => cubetuple_format_ptr(dno,error)
      if (error) return
      ! Set up an iterator suited for this cube
      ncube = ncube+1
      write(mess,'(A,I0)') '--- Computing iterator for cube #',ncube
      call cubeadm_message(admseve%others,rname,mess)
      call cubeadm_taskloop_cubedesc_fill(cube,.false.,null(),myiter(ncube),error)
      if (error) return
    enddo
    !
    ! Cubes consistency: number of entries must be the same or 1
    ! me = maxval(myiter(1:ncube)%ne)
    ! do icube=1,ncube
    !   if (myiter(icube)%ne.ne.1 .and. myiter(icube)%ne.ne.me) then
    !     write(mess,'(A,I0,A,I0)')  &
    !       'Iterated cubes have different number of entries: ',myiter(icube)%ne,',',me
    !     call cubeadm_message(seve%e,rname,mess)
    !     error = .true.
    !     return
    !   endif
    ! enddo
    !
    ! Compute the global iterator by merging all the individual ones
    call cubeadm_message(admseve%others,rname,'--- Computing global iterator')
    call cubeadm_taskloop_iterator_compute(myiter,ncube,align,iter,error)
    if (error) return
    call cubeadm_message(admseve%others,rname,'--- Done')
  end subroutine cubeadm_taskloop_init
  !
  subroutine cubeadm_taskloop_cubedesc_fill(cub,input,region,iter,error)
    use cubetools_access_types
    !----------------------------------------------------------------------
    ! Set up the iterator suited for a single cube.
    !----------------------------------------------------------------------
    class(format_t), target,           intent(in)    :: cub
    logical,                           intent(in)    :: input
    type(cuberegion_prog_t),           pointer       :: region
    type(cubeadm_iterator_cubedesc_t), intent(out)   :: iter
    logical,                           intent(inout) :: error
    !
    character(len=*), parameter :: rname='TASKLOOP>ITERATOR'
    !
    call cubeadm_message(admseve%trace,rname,'Welcome')
    !
    iter%input  = input
    iter%cube   => cub
    iter%region => region
    !
    ! Which access?
    select case (cub%access())
    case (code_access_subset,code_access_fullset)
      iter%access = cub%access()
    case (code_access_blobset)
      call cubeadm_message(seve%e,rname,'Blob access mode is not implemented')
      error = .true.
      return
    case default
      ! Other accesses based on actual data order
      select case (cub%order())
      case (code_access_imaset)
        iter%access = code_access_imaset
      case (code_access_speset)
        iter%access = code_access_speset
      case default
        call cubeadm_message(seve%e,rname,'Unsupported access mode')
        error = .true.
        return
      end select
    end select
    call cubeadm_message(admseve%others,rname,  &
      'Cube is ordered '//cubetools_accessname(cub%order())//  &
      ' and will be iterated with access '//cubetools_accessname(iter%access))
  end subroutine cubeadm_taskloop_cubedesc_fill
  !
  subroutine cubeadm_taskloop_iterator_compute(iter,ncube,align,merged,error)
    !-------------------------------------------------------------------
    ! Merge all the iterators to produce the final iterator
    !-------------------------------------------------------------------
    integer(kind=entr_k),              intent(in)    :: ncube
    type(cubeadm_iterator_cubedesc_t), intent(in)    :: iter(ncube)
    integer(kind=code_k),              intent(in)    :: align
    type(cubeadm_iterator_t),          intent(out)   :: merged
    logical,                           intent(inout) :: error
    !
    integer(kind=4) :: ref
    logical :: parallel
    real(kind=4) :: buftask
    character(len=message_length) :: mess
    character(len=*), parameter :: rname='TASKLOOP>MERGE'
    !
    ! This subroutine is the core of the problem. It is intended to
    ! scale properly the entries and tasks, knowing all the input and
    ! output cubes. The main goal is to compute the number of planes
    ! per task, which must fit in the user limit (max task size).
    !
    ! However there are some tricky details like mixed accesses. For
    ! example HISTO3D reads three 2D images (imaset) and creates one 2D
    ! image (imaset) and one 3D cube (fullset).
    !
    ! Because of this, fullset-accessed cubes are excluded in the analysis
    ! below as they are not involved in the task sizing (100% of data
    ! treated by a single task).
    !
    ! Select the reference cube
    ref = cubeadm_taskloop_get_reference(iter,ncube,error)
    if (error)  return
    !
    ! Set the reference axes to each iterated cube
    call cubeadm_tuple_iterator_axes(iter,ncube,iter(ref)%cube,align,error)
    if (error)  return
    !
    ! Search for best nplane-per-task
    parallel = .false.
    !$ parallel = .true.
    if (parallel .and. .not.iter(ref)%access.eq.code_access_fullset) then
      buftask = iter(ref)%cube%user%buff%task
    else
      buftask = huge(0.)
    endif
    call cubeadm_taskloop_nrpperbuf(iter,ncube,buftask,merged%nppertask,error)
    if (error)  return
    call cubeadm_taskloop_neperplane(iter(ref),merged%neperplane,error)
    if (error)  return
    merged%np = iter(ref)%cube%iter%compute_nplane_total(error)
    if (error)  return
    !
    ! Total number of tasks and entries. Can not guess by simple divisions.
    ! Apply iterator for exact values.
    call cubeadm_taskloop_reset(merged,error)
    if (error) return
    do while (cubeadm_taskloop_iterate(merged,error))
      continue
    enddo
    if (error)  return
    merged%nt = merged%num
    merged%ne = merged%ole
    !
    write(mess,'(4(a,i0))')   &
      'Split job using ',merged%nt, &
      ' tasks (',merged%nppertask, &
      ' planes per task) for ',merged%ne, &
      ' entries'
    call cubeadm_message(admseve%others,rname,mess)
    !
    ! Reset iteration before real use
    call cubeadm_taskloop_reset(merged,error)
    if (error) return
  end subroutine cubeadm_taskloop_iterator_compute
  !
  subroutine cubeadm_tuple_iterator_axes(iter,ncube,refcube,align,error)
    !-------------------------------------------------------------------
    ! Set the current and reference axes for the given cube
    !-------------------------------------------------------------------
    integer(kind=entr_k),              intent(in)    :: ncube
    type(cubeadm_iterator_cubedesc_t), intent(in)    :: iter(ncube)
    class(format_t),                   intent(in)    :: refcube
    integer(kind=code_k),              intent(in)    :: align
    logical,                           intent(inout) :: error
    !
    integer(kind=entr_k) :: iiter
    class(format_t), pointer :: thiscube
    type(cuberegion_prog_t), pointer :: thisregion
    !
    do iiter=1,ncube
      thiscube   => iter(iiter)%cube
      thisregion => iter(iiter)%region
      if (associated(thisregion)) then
        call thisregion%apply_to_cube(thiscube,error,align)
        if (error)  return
      else
        call thiscube%default_region(error,align)
        if (error)  return
      endif
      call thiscube%iter%set_reference(refcube%iter,error)
      if (error)  return
    enddo
  end subroutine cubeadm_tuple_iterator_axes
  !
  subroutine cubeadm_taskloop_neperplane(iter,neperplane,error)
    !-------------------------------------------------------------------
    !
    !-------------------------------------------------------------------
    type(cubeadm_iterator_cubedesc_t), intent(in)    :: iter
    integer(kind=entr_k),              intent(out)   :: neperplane
    logical,                           intent(inout) :: error
    !
    select case (iter%access)
    case (code_access_subset)
      neperplane = code_1entry_for_allplanes
    case (code_access_fullset)
      neperplane = code_1entry_for_allplanes
    case (code_access_imaset)
      neperplane = 1
    case (code_access_speset)
      neperplane = iter%cube%tuple%current%desc%nx
    end select
  end subroutine cubeadm_taskloop_neperplane
  !
  subroutine cubeadm_taskloop_nrpperbuf(iter,ncube,bufsize,nrpperbuf,error)
    use cubetools_format
    !-------------------------------------------------------------------
    ! Find the best (smallest) "Number of Reference Plane Per Block"
    !-------------------------------------------------------------------
    integer(kind=entr_k),              intent(in)    :: ncube
    type(cubeadm_iterator_cubedesc_t), intent(in)    :: iter(ncube)
    real(kind=4),                      intent(in)    :: bufsize  ! [Bytes]
    integer(kind=entr_k),              intent(out)   :: nrpperbuf
    logical,                           intent(inout) :: error
    !
    integer(kind=entr_k) :: iiter,thisnrp
    character(len=mess_l) :: mess
    character(len=*), parameter :: rname='TASKLOOP>NRPERBUF'
    !
    nrpperbuf = huge(0_entr_k)
    do iiter=1,ncube
      ! Fullset-accessed cubes are not involved in the best value
      if (iter(iiter)%access.eq.code_access_fullset)  cycle
      thisnrp = iter(iiter)%cube%iter%compute_nplane_per_buffer(bufsize,error)
      if (error)  return
      if (thisnrp.lt.nrpperbuf)  nrpperbuf = thisnrp
    enddo
    write(mess,'(i0,a,a)')  &
      nrpperbuf,' ref. planes per ',cubetools_format_memsize(bufsize)
    call cubeadm_message(admseve%others,rname,mess)
  end subroutine cubeadm_taskloop_nrpperbuf
  !
  function cubeadm_taskloop_get_reference(iter,ncube,error) result(ref_cube)
    !-------------------------------------------------------------------
    ! From the list of all cubes, choose the reference cube (i.e. used
    ! as reference for the iterations)
    !-------------------------------------------------------------------
    integer(kind=4) :: ref_cube  ! Function value on return
    integer(kind=entr_k),              intent(in)    :: ncube
    type(cubeadm_iterator_cubedesc_t), intent(in)    :: iter(ncube)
    logical,                           intent(inout) :: error
    !
    integer(kind=4) :: iiter
    character(len=mess_l) :: mess
    character(len=*), parameter :: rname='TASKLOOP>GET>REFERENCE'
    !
    ref_cube = 0
    !
    ! 1) Preferably select an output cube with non-fullset access
    do iiter=1,ncube
      if (iter(iiter)%input)                          cycle
      if (iter(iiter)%access.eq.code_access_fullset)  cycle
      ref_cube = iiter
      exit
    enddo
    if (ref_cube.ne.0)  goto 10
    !
    ! 2) Otherwise select an input cube with non-fullset access
    do iiter=1,ncube
      if (.not.iter(iiter)%input)                     cycle
      if (iter(iiter)%access.eq.code_access_fullset)  cycle
      ref_cube = iiter
      exit
    enddo
    if (ref_cube.ne.0)  goto 10
    !
    ! 3) Otherwise select an output cube possibly with fullset access
    do iiter=1,ncube
      if (iter(iiter)%input)                          cycle
      ref_cube = iiter
      exit
    enddo
    if (ref_cube.ne.0)  goto 10
    !
    ! 4) Otherwise select an input cube possibly with fullset access
    do iiter=1,ncube
      if (.not.iter(iiter)%input)                     cycle
      ref_cube = iiter
      exit
    enddo
    if (ref_cube.ne.0)  goto 10
    !
    call cubeadm_message(seve%e,rname,'Internal error: no cube reference cube found')
    error = .true.
    return
    !
10  continue
    write(mess,'(a,i0)')  'Reference cube is #',ref_cube
    call cubeadm_message(admseve%others,rname,mess)
  end function cubeadm_taskloop_get_reference
  !
  subroutine cubeadm_taskloop_reset(iter,error)
    !-------------------------------------------------------------------
    ! Reset iterator before iterating with cubeadm_taskloop_iterate
    !-------------------------------------------------------------------
    type(cubeadm_iterator_t), intent(inout) :: iter
    logical,                  intent(inout) :: error
    !
    iter%num = 0
    iter%prange(2) = 0  ! So that 1st iteration starts at correct position
  end subroutine cubeadm_taskloop_reset
  !
  function cubeadm_taskloop_iterate(iter,error)
    !----------------------------------------------------------------------
    ! Iterate the iterator for next range of entries to be processed.
    ! Return .false. if no more data is to be processed.
    !----------------------------------------------------------------------
    logical :: cubeadm_taskloop_iterate
    type(cubeadm_iterator_t), intent(inout) :: iter
    logical,                  intent(inout) :: error
    ! Local
    character(len=*), parameter :: rname='TASKLOOP>ITERATE'
    !
    if (.true.) then
      ! Plane-iterating mode
      cubeadm_taskloop_iterate = cubeadm_taskloop_iterate_planes(iter,error)
      if (error) return
    else
      ! Data-iterating mode
      ! cubeadm_taskloop_iterate = cubeadm_taskloop_iterate_data(iter,error)
      ! if (error) return
    endif
  end function cubeadm_taskloop_iterate
  !
  function cubeadm_taskloop_iterate_planes(iter,error)
    !----------------------------------------------------------------------
    ! Iterate the iterator for next range of PLANES to be processed.
    !
    ! Return .false. if no more data is to be processed.
    !----------------------------------------------------------------------
    logical :: cubeadm_taskloop_iterate_planes
    type(cubeadm_iterator_t), intent(inout) :: iter
    logical,                  intent(inout) :: error
    ! Local
    character(len=*), parameter :: rname='TASKLOOP>ITERATE>PLANES'
    integer(kind=entr_k) :: nplane,nentry
    character(len=message_length) :: mess
    !
    ! No welcome on purpose here!
    if (iter%prange(2).ge.iter%np) then
      ! Last entry iteration was processed on previous iteration: all done
      cubeadm_taskloop_iterate_planes = .false.
      return
    endif
    !
    ! Current task
    iter%num = iter%num+1
    !
    ! Reference planes
    iter%prange(1) = iter%prange(2)+1
    iter%prange(2) = min(iter%prange(1)+iter%nppertask-1,iter%np)
    nplane = iter%prange(2)-iter%prange(1)+1
    !
    ! Reference entries
    if (iter%neperplane.eq.code_1entry_for_allplanes) then
      ! 1 subcube for all planes in the task
      iter%ofe = iter%num
      iter%ole = iter%ofe
    else
      iter%ofe = (iter%num-1)*iter%nppertask*iter%neperplane+1
      iter%ole = iter%ofe + nplane*iter%neperplane - 1
    endif
    !
    ! Entries
    nentry = iter%ole-iter%ofe+1
    iter%ie = iter%ofe-1  ! Current entry, to be iterated with cubeadm_iterator_iterate_entry
    !
    cubeadm_taskloop_iterate_planes = .true.
    !
    write(mess,'(8(a,i0))')  &
      'Iterating task ',iter%num,  &
      ', entries ',iter%ofe,  &
      ' to ',iter%ole,  &
      ' (',nentry, &
      ' entries), planes ',iter%prange(1),  &
      ' to ',iter%prange(2),  &
      ' (',nplane, &
      ' planes)'
    call cubeadm_message(admseve%others,rname,mess)
  end function cubeadm_taskloop_iterate_planes
  !
  function cubeadm_iterator_iterate_entry(itertask,error)
    !-------------------------------------------------------------------
    ! Iterate the entry known by the current task iterator
    !-------------------------------------------------------------------
    logical :: cubeadm_iterator_iterate_entry
    class(cubeadm_iterator_t), intent(inout) :: itertask
    logical,                   intent(inout) :: error
    !
    cubeadm_iterator_iterate_entry = .false.
    !
    if (itertask%ie.eq.itertask%ole)  return  ! All done
    !
    itertask%ie = itertask%ie+1
    call cubeadm_entryloop_iterate(itertask%ie,error)
    if (error)  return
    cubeadm_iterator_iterate_entry = .true.  ! New entry and no error
  end function cubeadm_iterator_iterate_entry
  !
end module cubeadm_taskloop
!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
