HPL 源码结构分析

文件夹结构：

$ cd /home/hipper/ex_hpl_hpcg/

$ pwd

$ mkdir ./openmpi

$mkdir ./openblas

$mkdir ./hpl

$ tree

1. 安装openmpi

1.1.1 使用Makefile下载配置编译安装 openmpi

Makefile：

all:wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.gz && \tar zxf openmpi-4.1.6.tar.gz && \cd openmpi-4.1.6/ && \./configure --enable-mpi-cxx --with-cuda --prefix=${PWD}/local/ 2>&1 | tee config.out && \make -j all 2>&1 | tee make.outinstall:make -C openmpi-4.1.6/ install 2>&1 | tee install.out.PHONY: clean
clean:-rm -rf ./openmpi-4.1.6/ ./openmpi-4.1.6.tar.gz

$ make
$ make install

$ ls

1.1.2 注意事项

如果 是 openmpi 5.0及其以上，则./configure 时不能要选项：

--enable-mpi-cxx

mpi 以后仅支持使用相对广泛很多的 C 语言风格的 API，不再支持 C++ 风格的使用方式，虽然有在用，但很不主流，C 风格 api 足够强大，足够使用。

无论4.x还是5.x，如果没有安装cuda，则 ./configure 时不需要该项：

--with-cuda

而，指定安装目录时需要指定绝对路径，例如：

--prefix=${PWD}/local/

或者不使用 Makefile 方式，而使用命令行安装 openmpi：

$ wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.g
$ tar zxf openmpi-4.1.6.tar.gz
$ cd openmpi-4.1.6/其中 configure 选项 --prefix=/.../ 需要使用绝对路径，例如：
$ ./configure --enable-mpi-cxx --with-cuda --prefix=/home/hipper/ex_hpl/openmpi/local/ 2>&1 | tee config.out
$ make -j all 2>&1 | tee make.out
$ make install 2>&1 | tee install.out

1.1.3 配置环境变量

 export PATH=/home/hipper/ex_openmpi/local/bin:$PATHexport LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/hipper/ex_openmpi/local/libcd examplesmakempirun -np 7 hello_c

2. 安装 openblas

 
all:wget https://github.com/OpenMathLib/OpenBLAS/archive/refs/tags/v0.3.27.tar.gztar zxf v0.3.27.tar.gz make -C OpenBLAS-0.3.27 FC=gfortran -jinstall:make -C OpenBLAS-0.3.27 install PREFIX=../local/
#PREFIX=/opt/OpenBLAS
# PREFIX=../local/clean:-rm -rf ./local/ ./OpenBLAS-0.3.27/ ./v0.3.27.tar.gz

$ make
$ make install
$ ls

3. 配置编译运行 hpl

1.3.1 下载 hpl 源代码

$ wget https://www.netlib.org/benchmark/hpl/hpl-2.3.tar.gz
$ tar zxf hpl-2.3.tar.gz
$ cd hpl-2.3/

1.3.2 配置

$ cd hpl-2.3/
$ hpl-2.3$ cp ./setup/Make.Linux_PII_CBLAS ./

vim ./Make.Linux_PII_CBLAS

# ######################################################################
#
# ----------------------------------------------------------------------
# - shell --------------------------------------------------------------
# ----------------------------------------------------------------------
#
SHELL        = /bin/sh
#
CD           = cd
CP           = cp
LN_S         = ln -s
MKDIR        = mkdir
RM           = /bin/rm -f
TOUCH        = touch
#
# ----------------------------------------------------------------------
# - Platform identifier ------------------------------------------------
# ----------------------------------------------------------------------
#
ARCH         = Linux_PII_CBLAS
#
# ----------------------------------------------------------------------
# - HPL Directory Structure / HPL library ------------------------------
# ----------------------------------------------------------------------
#
#/home/hipper/ex_hpl_hpcg/hpl/hpl-2.3
TOPdir       = $(HOME)/ex_hpl_hpcg/hpl/hpl-2.3
INCdir       = $(TOPdir)/include
BINdir       = $(TOPdir)/bin/$(ARCH)
LIBdir       = $(TOPdir)/lib/$(ARCH)
#
HPLlib       = $(LIBdir)/libhpl.a
#
# ----------------------------------------------------------------------
# - Message Passing library (MPI) --------------------------------------
# ----------------------------------------------------------------------
# MPinc tells the  C  compiler where to find the Message Passing library
# header files,  MPlib  is defined  to be the name of  the library to be
# used. The variable MPdir is only used for defining MPinc and MPlib.
#
#MPdir        = /usr/local/mpi
MPdir        = $(HOME)/ex_hpl_hpcg/openmpi/local
MPinc        = -I$(MPdir)/include
MPlib        = $(MPdir)/lib/libmpi.so
#
# ----------------------------------------------------------------------
# - Linear Algebra library (BLAS or VSIPL) -----------------------------
# ----------------------------------------------------------------------
# LAinc tells the  C  compiler where to find the Linear Algebra  library
# header files,  LAlib  is defined  to be the name of  the library to be
# used. The variable LAdir is only used for defining LAinc and LAlib.
#
#LAdir        = $(HOME)/netlib/ARCHIVES/Linux_PII
LAdir        = $(HOME)/ex_hpl_hpcg/openblas/local
LAinc        =
LAlib        = $(LAdir)/lib/libopenblas.a
#
# ----------------------------------------------------------------------
# - F77 / C interface --------------------------------------------------
# ----------------------------------------------------------------------
# You can skip this section  if and only if  you are not planning to use
# a  BLAS  library featuring a Fortran 77 interface.  Otherwise,  it  is
# necessary  to  fill out the  F2CDEFS  variable  with  the  appropriate
# options.  **One and only one**  option should be chosen in **each** of
# the 3 following categories:
#
# 1) name space (How C calls a Fortran 77 routine)
#
# -DAdd_              : all lower case and a suffixed underscore  (Suns,
#                       Intel, ...),                           [default]
# -DNoChange          : all lower case (IBM RS6000),
# -DUpCase            : all upper case (Cray),
# -DAdd__             : the FORTRAN compiler in use is f2c.
#
# 2) C and Fortran 77 integer mapping
#
# -DF77_INTEGER=int   : Fortran 77 INTEGER is a C int,         [default]
# -DF77_INTEGER=long  : Fortran 77 INTEGER is a C long,
# -DF77_INTEGER=short : Fortran 77 INTEGER is a C short.
#
# 3) Fortran 77 string handling
#
# -DStringSunStyle    : The string address is passed at the string loca-
#                       tion on the stack, and the string length is then
#                       passed as  an  F77_INTEGER  after  all  explicit
#                       stack arguments,                       [default]
# -DStringStructPtr   : The address  of  a  structure  is  passed  by  a
#                       Fortran 77  string,  and the structure is of the
#                       form: struct {char *cp; F77_INTEGER len;},
# -DStringStructVal   : A structure is passed by value for each  Fortran
#                       77 string,  and  the  structure is  of the form:
#                       struct {char *cp; F77_INTEGER len;},
# -DStringCrayStyle   : Special option for  Cray  machines,  which  uses
#                       Cray  fcd  (fortran  character  descriptor)  for
#                       interoperation.
#
F2CDEFS      =
#
# ----------------------------------------------------------------------
# - HPL includes / libraries / specifics -------------------------------
# ----------------------------------------------------------------------
#
HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) $(LAinc) $(MPinc) -lpthread
HPL_LIBS     = $(HPLlib) $(LAlib) $(MPlib) -lpthread
#
# - Compile time options -----------------------------------------------
#
# -DHPL_COPY_L           force the copy of the panel L before bcast;
# -DHPL_CALL_CBLAS       call the cblas interface;
# -DHPL_CALL_VSIPL       call the vsip  library;
# -DHPL_DETAILED_TIMING  enable detailed timers;
#
# By default HPL will:
#    *) not copy L before broadcast,
#    *) call the BLAS Fortran 77 interface,
#    *) not display detailed timing information.
#
HPL_OPTS     = -DHPL_CALL_CBLAS
#
# ----------------------------------------------------------------------
#
HPL_DEFS     = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)
#
# ----------------------------------------------------------------------
# - Compilers / linkers - Optimization flags ---------------------------
# ----------------------------------------------------------------------
#
CC           = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpicc
CCNOOPT      = $(HPL_DEFS)
CCFLAGS      = $(HPL_DEFS) -fomit-frame-pointer -O3 -funroll-loops
#
# On some platforms,  it is necessary  to use the Fortran linker to find
# the Fortran internals used in the BLAS library.
#
LINKER       = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpif77
LINKFLAGS    = $(CCFLAGS)
#
ARCHIVER     = ar
ARFLAGS      = r
RANLIB       = echo
#
# ----------------------------------------------------------------------

前后区别：

$ diff Make.Linux_PII_CBLAS setup/Make.Linux_PII_CBLAS 

hipper@hipper-G21:~/ex_hpl_hpcg/hpl/hpl-2.3$ diff Make.Linux_PII_CBLAS setup/Make.Linux_PII_CBLAS
70,71c70
< #/home/hipper/ex_hpl_hpcg/hpl/hpl-2.3
< TOPdir       = $(HOME)/ex_hpl_hpcg/hpl/hpl-2.3
---
> TOPdir       = $(HOME)/hpl
85,86c84
< #MPdir        = /usr/local/mpi
< MPdir        = /home/hipper/ex_hpl_hpcg/openmpi/local
---
> MPdir        = /usr/local/mpi
88c86
< MPlib        = $(MPdir)/lib/libmpi.so
---
> MPlib        = $(MPdir)/lib/libmpich.a
97,98c95
< #LAdir        = $(HOME)/netlib/ARCHIVES/Linux_PII
< LAdir        = $(HOME)/ex_hpl_hpcg/openblas/local
---
> LAdir        = $(HOME)/netlib/ARCHIVES/Linux_PII
100c97
< LAlib        = $(LAdir)/lib/libopenblas.a
---
> LAlib        = $(LAdir)/libcblas.a $(LAdir)/libatlas.a
147,148c144,145
< HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) $(LAinc) $(MPinc) -lpthread
< HPL_LIBS     = $(HPLlib) $(LAlib) $(MPlib) -lpthread
---
> HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) $(LAinc) $(MPinc)
> HPL_LIBS     = $(HPLlib) $(LAlib) $(MPlib)
172c169
< CC           = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpicc
---
> CC           = /usr/bin/gcc
179c176
< LINKER       = $(HOME)/ex_hpl_hpcg/openmpi/local/bin/mpif77
---
> LINKER       = /usr/bin/g77
hipper@hipper-G21:~/ex_hpl_hpcg/hpl/hpl-2.3$

1.3.3编译运行

make arch=Linux_PII_CBLAS

$ ls

单节点运行：

$ /home/hipper/ex_hpl_hpcg/openmpi/local/bin/mpirun -np 4 ./xhpl > HPL-Benchmark.txt

效果：
多节点：
mpirun -np 8 --host node1,node2 ./xhpl > HPL-Benchmark.txt

https://www.advancedclustering.com/act_kb/tune-hpl-dat-file
mpirun 的参数 np 值需要跟 HPL.dat 的 (P x Q) 相等。

2，hpl 源码框架分析

先保存备忘，待续。。。。