Hi,
Installed latest 6.2 on a VM with no access to SAN and added my nodes to the BRIDGE_LIST.
My three KVM’s has two shared LUNs as id 102 and 103 with vgs created as vg-one-102 and vg-one-103.
Created a SYSTEM_DS and IMAGE_DS using the following definitions.
SYSTEM
-sh-4.2$ cat ds01.conf
NAME = lvm_system
TM_MAD = fs_lvm_ssh
TYPE = SYSTEM_DS
BRIDGE_LIST = "kvm116.example.com kvm117.example.com kvm118.example.com"
<ID 102>
IMAGE
-sh-4.2$ cat ds02.conf
NAME = production
DS_MAD = fs
TM_MAD = fs_lvm_ssh
DISK_TYPE = "BLOCK"
TYPE = IMAGE_DS
SAFE_DIRS="/var/tmp /tmp"
BRIDGE_LIST = "kvm116.example.com kvm117.example.com kvm118.example.com"
Getting the following issue while trying to instanciate a vm.
Sat Apr 2 18:01:39 2022 [Z0][TrM][D]: Message received: TRANSFER FAILURE 20 + SRC=one-01.example.com:/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 + DST=kvm116.example.com:/var/lib/one//datastores/102/20/disk.0 + VM_ID=20 + DS_ID=103 + '[' -z '' ']' + TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh ++ dirname /var/lib/one/remotes/tm/fs_lvm_ssh/clone + DRIVER_PATH=/var/lib/one/remotes/tm/fs_lvm_ssh + source /var/lib/one/remotes/tm/tm_common.sh ++ export LANG=C ++ LANG=C ++ '[' -z '' ']' ++ ONE_LOCAL_VAR=/var/lib/one ++ ONE_LIB=/usr/lib/one ++ DS_DIR=/var/lib/one/datastores ++ ONE_SH=/usr/lib/one/sh ++ . /usr/lib/one/sh/scripts_common.sh +++ export LANG=C +++ LANG=C +++ export PATH=/bin:/sbin:/usr/bin:/usr/lib/one/sh/override:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ PATH=/bin:/sbin:/usr/bin:/usr/lib/one/sh/override:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +++ AWK=awk +++ BASH=/bin/bash +++ CUT=cut +++ CEPH=ceph +++ DATE=date +++ DD=dd +++ DF=df +++ DU=du +++ GREP=grep +++ ISCSIADM=iscsiadm +++ LVCREATE=lvcreate +++ LVREMOVE=lvremove +++ LVCHANGE=lvchange +++ LVSCAN=lvscan +++ LVEXTEND=lvextend +++ LVS=lvs +++ LN=ln +++ MD5SUM=md5sum +++ MKFS=mkfs +++ MKISOFS=genisoimage +++ MKSWAP=mkswap +++ QEMU_IMG=qemu-img +++ RADOS=rados +++ RBD=rbd +++ READLINK=readlink +++ RM=rm +++ CP=cp +++ SED=sed +++ SSH=ssh +++ SSH_FWD='ssh -o ForwardAgent=yes -o ControlMaster=no -o ControlPath=none' +++ SUDO='sudo -n' +++ SYNC=sync +++ TAR=tar +++ TGTADM=tgtadm +++ TGTADMIN=tgt-admin +++ TGTSETUPLUN=tgt-setup-lun-one +++ TR=tr +++ VGDISPLAY=vgdisplay +++ VMKFSTOOLS=vmkfstools +++ WGET=wget ++++ uname -s +++ '[' xLinux = xLinux ']' +++ SED='sed -r' ++++ basename -- /var/lib/one/remotes/tm/fs_lvm_ssh/clone +++ SCRIPT_NAME=clone + source /var/lib/one/remotes/tm/fs_lvm_ssh/../../datastore/libfs.sh + source /var/lib/one/remotes/tm/fs_lvm_ssh/../../etc/tm/fs_lvm/fs_lvm.conf ++ ZERO_LVM_ON_CREATE=yes ++ ZERO_LVM_ON_DELETE=yes ++ DD_BLOCK_SIZE=64k ++ arg_path kvm116.example.com:/var/lib/one//datastores/102/20/disk.0 +++ echo kvm116.example.com:/var/lib/one//datastores/102/20/disk.0 +++ sed -r 's/^[^:]*:(.*)$/\1/' ++ ARG_PATH=/var/lib/one//datastores/102/20/disk.0 ++ '[' '' = NOFIX ']' ++ fix_dir_slashes /var/lib/one//datastores/102/20/disk.0 ++ dirname /var/lib/one//datastores/102/20/disk.0/file ++ sed -r 's/\/\+/\//g' + DST_PATH=/var/lib/one//datastores/102/20/disk.0 ++ arg_host kvm116.example.com:/var/lib/one//datastores/102/20/disk.0 ++ echo kvm116.example.com:/var/lib/one//datastores/102/20/disk.0 ++ sed -r 's/^([^:]*):.*$/\1/' + DST_HOST=kvm116.example.com ++ dirname /var/lib/one//datastores/102/20/disk.0 + DST_DIR=/var/lib/one//datastores/102/20 ++ arg_path one-01.example.com:/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 +++ echo one-01.example.com:/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 +++ sed -r 's/^[^:]*:(.*)$/\1/' ++ ARG_PATH=/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ '[' '' = NOFIX ']' ++ fix_dir_slashes /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ dirname /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045/file ++ sed -r 's/\/\+/\//g' + SRC_ARG_PATH=/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++++ dirname /var/lib/one//datastores/102/20/disk.0 +++ dirname /var/lib/one//datastores/102/20 ++ dirname /var/lib/one//datastores/102 + DST_DS_PATH=/var/lib/one//datastores +++ dirname /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ dirname /var/lib/one//datastores/103 + SRC_DS_PATH=/var/lib/one//datastores + SRC_PATH=/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ arg_host one-01.example.com:/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ echo one-01.example.com:/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ sed -r 's/^([^:]*):.*$/\1/' + SRC_HOST=one-01.example.com ++ echo /var/lib/one//datastores/102/20 ++ awk -F / '{print $(NF-1)}' + DS_SYS_ID=102 ++ basename /var/lib/one//datastores/102/20/disk.0 ++ cut -d. -f2 + DISK_ID=0 + XPATH='/var/lib/one/remotes/tm/fs_lvm_ssh/../../datastore/xpath.rb --stdin' + unset i j XPATH_ELEMENTS + IFS= + read -r -d '' element ++ onevm show -x 20 ++ /var/lib/one/remotes/tm/fs_lvm_ssh/../../datastore/xpath.rb --stdin '/VM/TEMPLATE/DISK[DISK_ID=0]/SIZE' '/VM/TEMPLATE/DISK[DISK_ID=0]/ORIGINAL_SIZE' '/VM/HISTORY_RECORDS/HISTORY[last()]/TM_MAD' + XPATH_ELEMENTS[i++]=4096 + IFS= + read -r -d '' element + XPATH_ELEMENTS[i++]=4096 + IFS= + read -r -d '' element + XPATH_ELEMENTS[i++]=fs_lvm_ssh + IFS= + read -r -d '' element + SIZE=4096 + ORIGINAL_SIZE=4096 + TM_MAD=fs_lvm_ssh ++ fs_size /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 YES ++ SRC=/var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 +++ echo YES +++ tr A-Z a-z ++ NO_DECOMPRESS=yes ++ LIMIT_TRANSFER_BW= +++ set_downloader_args '' '' yes '' +++ HASHES=' ' +++ '[' -n '' ']' +++ '[' -n '' ']' ++++ echo yes ++++ tr A-Z a-z +++ '[' xyes = xyes ']' +++ HASHES=' --nodecomp' +++ '[' -n '' ']' +++ '[' -n ' --nodecomp' ']' +++ echo -ne ' --nodecomp ' +++ '[' -n '' ']' +++ '[' -n '' ']' ++ DOWNLOADER_ARGS=' --nodecomp ' ++ '[' -z '' ']' ++ '[' -n /var/lib/one/remotes/tm/fs_lvm_ssh ']' ++ UTILS_PATH=/var/lib/one/remotes/tm/fs_lvm_ssh/../../datastore ++ '[' -d /var/lib/one/remotes/tm/fs_lvm_ssh/../../datastore ']' ++ error=1 ++ '[' -d /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ']' ++ echo /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ grep -qe '^docker\?://\|^dockerfile\?://' ++ '[' -f /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ']' ++ echo /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 ++ grep -qe '^https\?://' ++ echo 'File not found' ++ return 1 + FILE_SIZE='File not found' ++ file --brief /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 + FILE_TYPE='cannot open (No such file or directory)' + '[' 0 -ne 0 ']' + '[' File not found -gt 4096 ']' /var/lib/one/remotes/tm/fs_lvm_ssh/clone: line 89: [: too many arguments + [[ cannot open (No such file or directory) =~ LUKS encrypted file ]] + '[' 'File not found' -lt 4096 ']' /var/lib/one/remotes/tm/fs_lvm_ssh/clone: line 101: [: File not found: integer expression expected + ZERO_SEEK_BYTES=4096 + '[' 4096 -gt 0 ']' + ZERO_SEEK_BYTES=4293918720 + LV_NAME=lv-one-20-0 + VG_NAME=vg-one-102 + DEV=/dev/vg-one-102/lv-one-20-0 ++ cat + CREATE_CMD=' set -e -o pipefail sync sudo -n lvscan sudo -n lvcreate --wipesignatures n -L4096M -n lv-one-20-0 vg-one-102' + LOCK=tm-fs_lvm-102.lock + exclusive tm-fs_lvm-102.lock 120 ssh_exec_and_log kvm116.example.com ' set -e -o pipefail sync sudo -n lvscan sudo -n lvcreate --wipesignatures n -L4096M -n lv-one-20-0 vg-one-102' 'Error creating LV named lv-one-20-0' + LOCK_FILE=/var/lock/one/tm-fs_lvm-102.lock + TIMEOUT=120 + shift 2 + umask 0027 + touch /var/lock/one/tm-fs_lvm-102.lock + '[' 0 -ne 0 ']' + flock -w 120 11 + '[' 0 -ne 0 ']' + ssh_exec_and_log kvm116.example.com ' set -e -o pipefail sync sudo -n lvscan sudo -n lvcreate --wipesignatures n -L4096M -n lv-one-20-0 vg-one-102' 'Error creating LV named lv-one-20-0' + ssh_exec_and_log_no_error kvm116.example.com ' set -e -o pipefail sync sudo -n lvscan sudo -n lvcreate --wipesignatures n -L4096M -n lv-one-20-0 vg-one-102' 'Error creating LV named lv-one-20-0' ++ ssh kvm116.example.com bash -s + SSH_EXEC_ERR= + SSH_EXEC_RC=0 + '[' 0 -ne 0 ']' + return 0 + EXEC_RC=0 + '[' 0 '!=' 0 ']' + EXEC_RC=0 + eval 'exec 11>&-' ++ exec + return 0 ++ cat + CLONE_CMD=' set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"' + ssh_forward ssh_exec_and_log kvm116.example.com ' set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"' 'Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0' + _ssh_cmd_saved=ssh + SSH='ssh -o ForwardAgent=yes -o ControlMaster=no -o ControlPath=none' + ssh_exec_and_log kvm116.example.com ' set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"' 'Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0' + ssh_exec_and_log_no_error kvm116.example.com ' set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"' 'Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0' ++ ssh -o ForwardAgent=yes -o ControlMaster=no -o ControlPath=none kvm116.example.com bash -s + SSH_EX' + SSH_EXEC_RC=2 + '[' 2 -ne 0 ']' + '[' 2 '!=' '' ']' + log_error 'Command " set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"" failed: 16+0 records in 16+0 records ' + log_function ERROR 'Command " set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"" failed: 16+0 records in 16+0 records out 1048576 bytes (1.0 MB, 1.0 MiB) copied, 0' + echo 'ERROR: clone: Command " set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '\''[:blank:]'\'') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"" failed: 16+0 records in 16+0 records out 1048576 bytes (1.0 MB, 1.0 MiB) copied, 0' ERROR: clone: Command " set -e -o pipefail mkdir -p /var/lib/one//datastores/102/20 hostname -f >"/var/lib/one//datastores/102/20/.host" || : # zero trailing space if [ "yes" = "yes" ]; then LVSIZE=$(sudo -n lvs --nosuffix --noheadings --units B -o lv_size "/dev/vg-one-102/lv-one-20-0" | tr -d '[:blank:]') dd if=/dev/zero of="/dev/vg-one-102/lv-one-20-0" bs=64k oflag=seek_bytes iflag=count_bytes seek="4293918720" count="$(( LVSIZE - 4293918720 ))" fi rm -f "/var/lib/one//datastores/102/20/disk.0" ln -s "/dev/vg-one-102/lv-one-20-0" "/var/lib/one//datastores/102/20/disk.0" ssh -n one-01.example.com "tar -cSO /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 2> /dev/null" | tar -xO 2> /dev/null > "/dev/vg-one-102/lv-one-20-0"" failed: 16+0 records in 16+0 records out 1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00217322 s, 4 + '[' -n 'Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0' ']' + error_message 'Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0' + echo 'Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0' Error cloning /var/lib/one//datastores/103/1d3697efb69993270345d910882eb045 to lv-one-20-0 + return 2 + EXEC_RC=2 + '[' 2 '!=' 0 ']' + exit 2```