The Cabin in the City

Properties Resource

2022-08-22 Spring

PathMatchingResourcePatternResolver resourceResolver = new PathMatchingResourcePatternResolver();

Resource[] rss = resourceResolver.getResources("classpath*/errorcode/errorCode*.properties");

PropertiesFactoryBean codeTab = new PropertiesFactoryBean();
codeTab.setLocations(rss);

return codeTab;

backup

2022-08-21 Linux

使用 WSL 进行 Windows 下不同磁盘之间的文件备份

modified.sh

#!/bin/bash

sourceDir=$1
find $1 -type f -mmin -60 | \
awk \
'BEGIN { "date \"+%F %T\""| getline; print $0 } \
{ print "modified:"$0 } \
END { print "\n\n"}'

backup.sh

#!/bin/bash
# copy source to target
#
# system(cmdrm);
baseDir="/mnt/e/SoftConfigBackup"
awk -F"=" '!/^\s*#/  \
{ origDir=$1; \
cmd0=bd"/modified.sh "$1" >> "bd"/modified.log"; \
system(cmd0); \
sub("^.*/", "", origDir); \
cmdrm="rm -rf "$2"/"origDir; \
cmd1="mkdir -p "$2; \
cmd2="cp -r " $1 " " $2; \
system(cmd1); \
system(cmd2) }' \
    bd=$baseDir "$baseDir/backupToNewDisk.properties"

date >>$baseDir/date.txt

1
2
3

crontab -l

*/60 * * * * /mnt/e/SoftConfigBackup/backup.sh

文件目录大小排序

#!/usr/bin/env bash

# 当前目录下，根据文件或目录的大小进行降序排序, 需要有对应目录的权限

# set current directory as default directory
dir=${1:-$(pwd)}
dir=$(echo $dir | sed -E 's/\/$//g')

IFSOld=$IFS
IFS=$'\n'
rs=""
for i in $(ls "${dir}"); do
    fpath="${dir}/$i"
    if [ -f "$fpath" ]; then
        size=$(du -B 1G "$fpath" | cut -f1)
        rs=$rs"\n[f]\x1c${size}\x1c$fpath"
    elif [ -d "$fpath" ]; then
        size=$(du -B 1G -c "$fpath" | grep total | cut -f1)
        rs=$rs"\n[d]\x1c${size}\x1c$fpath"
    fi
done
echo "block size = 1GB"
echo -e "\nFLG Block Path"
echo -e "-----------------------------------------------"
echo -e $rs | sort -t $'\x1c' -n -r -k2 | tr $'\x1c' ' '
IFS=$IFSOld

it also works

1	sep=$(echo -n -e '\x1c'); echo -e '2\x1cB\n1\x1cD' \| sort -t $sep -n -k1

1 2	cat /dev/null > testfile # 由于/dev/null文件不含有任何内容，通常用它来快速清除现有文件中的数据，而不用先删除文件再重新创建

重定向

$ IFS=','; while read name age; do cat >> p.sql << EOF
> insert into table (name, age) values ('$name', '$age');
> EOF
> done < a.csv

$ cat p.sql
insert into table (name, age) values ('Jason', '24');
insert into table (name, age) values ('Guo', '29');

$ cat a.csv
Jason,24
Guo,29

Bash脚本基本语法

2022-08-20 Linux

IFS=$'\n'

array.sh

#!/bin/bash

names=("Jason Chen" "Bell" "Alice")

# print all array members
echo "${names[@]}"

newNames=("${names[@]}" "Jason Guo")
echo "${newNames[@]}"


days=([0]=Sun [1]=Mon [2]=Tue [3]=Wed [4]=Thu [5]=Fri [6]=Sta)
echo "days: ${#days[@]}"
echo "value(index at 0): ${days[0]}"
echo "value's length at index 0: ${days[0]}"

## return indexes
for i in ${!days[@]}; do
  echo "index: $i, value: ${days[$i]}"
done

## extract array members: ${array[@]:position:length}
echo ${days[@]:1:3}

## append to array
days+=("Unk" Unk"")
echo "${days[@]}"

## del a member from array
unset days[7]
days[8]=''
echo "after remove index[7,8]: ${days[@]}"

## empty array
unset days
echo "after empty array: ${days[@]}"


echo "your input will be added to array:"
read -a dice

for i in "${dice[@]}"; do
  echo $i
done


declare -A colors
colors["red"]="#ff0000"

echo ${colors["red"]}

bashEnv.sh

#!/bin/bash

function func1() {
  ## 打印函数调用栈 FUNCNAME 返回一个数组，0号元素是当前调用的函数
  ## 1 号元素是调用当前函数的函数，以此类推
  echo -e "\nfunction call stack:"
  echo "FUNCNAME[0]: ${FUNCNAME[0]}"
  echo "FUNCNAME[1]: ${FUNCNAME[1]}"
}

echo "This is ${LINENO}th line in this script file"
func1

custom_env.sh

#!/usr/bin/env bash
export CUST_ENV4=guo
CUST_ENV5=peng
CUST_ENV6=hui

(echo $CUST_ENV4)

hello.sh

#!/bin/bash

hello="hello world"
echo "$hello, Tang!"

if.sh

#!/bin/bash

echo -n "plz input your answer:"
read ans

### String condition test
if [ -z "$ans" ]; then
        echo "you input nothing!"
        exit 1
fi

if [ "$ans" = "yes" ]; then
        echo "Yeah!!!!!!!!"
elif [ "$ans" = "no" ]; then
        echo "Sorry to hear that!!!!!!!"
elif [ "$ans" = "maybe" ]; then
        echo "You have make up your mind .-."
else
        echo "unknow answer"
fi

echo -n "plz input a num:"
read INT

### [[ ]] supports regex
if [[ "$INT" =~ ^-?[0-9]+$ ]]; then
        echo "yeah, you input a number, good boy!"
else
        echo "your input is not a number!"
    exit 1
fi

### integer condition
if [ $INT -eq 0 ]; then
        echo "zero"
elif [ $INT -gt 0 ]; then
        echo "great than zero"
else
        echo "less than zero"
fi

echo -n 'if ((3%2)) result='
if ((3%2)); then echo "true"; else echo "false"; fi

echo -n 'if ((3%3)) result='
if ((3%3)); then echo "true"; else echo "false"; fi

echo -n 'if ((3 > 2)) result='
if ((3 > 2)); then echo "true"; else echo "false"; fi

echo -n 'if ((3 < 2)) result='
if ((3 < 2)); then echo "true"; else echo "false"; fi

echo -n 'if ((3 == 3)) result='
if ((3 == 3)); then echo "true"; else echo "false"; fi

## command1 || command2; if command1 is false, then run command2
[ -d dir ] || mkdir dir

loop.sh

#!/bin/bash

## loop test

number=1
while [ "$number" -le 10 ]; do
        echo "$number hello"
        number=$((number+1))
done

for i in *.sh; do
        echo "shell file: $i"
done

## break continue
for (( i=0; i<5; i=i+1 )); do
        echo $i
        if [ $i -eq 3 ]; then
                echo "break statement"
                break
                # continue
        fi
done

month.sh

#!/bin/bash

#echo "input month number:"
#read monthNum

monthNum=$1
if [[ "$monthNum" =~ ^(1[0-2]|0?[1-9])$ ]]; then
        echo -n "you input $monthNum, and it is: "
        case $monthNum in
        1 | 01) echo "Jan" ;;
        2 | 02) echo "Feb" ;;
        3 | 03) echo "Mar" ;;
        4 | 04) echo "Apr" ;;
        5 | 05) echo "May" ;;
        6 | 06) echo "Jun" ;;
        7 | 07) echo "Jul" ;;
        8 | 08) echo "Aug" ;;
        9 | 09) echo "Sep" ;;
        10) echo "Oct" ;;
        11) echo "Nov" ;;
        12) echo "Dec" ;;
        esac
fi

parameter-parse.sh

#!/usr/bin/env bash
$(echo 'command')

echo $PWD

# 当前脚本中没有定义该变量，但如果调用脚本的父shell定义了该变量，并使用 source 执行，则可以打印
echo -e "\nvariable defined in parent shell: $CUST_ENV1\n"

echo 时间: $(date '+%F %T')
echo 带有option的参数列表: $@
echo 带有option的参数个数: $#


while getopts 'sn:' OPT; do
  case "$OPT" in
        s)
        echo "silent output enabled"
        ;;

        n)
        echo "output number: $OPTARG"
        ;;
        ?)
        echo "avaliable options: [-s] [-n number]" >&2 ## standard error
        exit 1
        ;;
  esac
done
shift "$(($OPTIND - 1))"

##
echo "option 选项处理完成，处理剩余主参数列表: $@"

## for 循环遍历参数列表
echo -e "\nfor 循环便利参数列表"
for i in $@; do
  echo 参数：$i
done


echo -e "\nshift 遍历参数列表：$@"
## shift n 移除脚本当前n个参数, 没有指定默认为1；
while [ "$1" != "" ]; do
  echo 参数个数：$#
  echo 参数：$1
  shift
done

readFile.sh

#!/usr/bin/env bash

if [ $# == 0 ]; then
  echo "请指定文件路径"
  exit 2
fi

for i in "$@"; do
    echo -e "\nfile: $i"
    num=1
    while read line; do
        echo -e "$((num++))\t$line"
    done < $i
done

readInput.sh

#!/usr/bin/env bash
echo -n "输入您的用户名 > "
read user
read -s -p "请输入密码 > " passwd1
echo
read -s -p "请确认密码 > " passwd2
echo

if [ "$passwd1" == "$passwd2" ]; then
        echo "注册成功, 用户名: '$user', 密码: '$passwd1'"
else
        echo "密码输入不一致: '$passwd1', '$passwd2'"
fi

select.sh

#!/bin/bash

echo "select the seq number to choose(use ctrl+c to quit)"

select gender in male female other
do
        case $gender in
                "male")   echo "Hi man" ;;
                "female") echo "Hi girl" ;;
                "other")  echo "Hi there" ;;
                *)        echo "Unknow seq" ;;
        esac
done

setShellDisallowNoUnset.sh

#!/bin/bash

# set -o nounset: none set variable will case exit
set -u

echo $a
echo "hi"

setShellExitOnError.sh

#!/bin/bash

# 遇到错误退出可以使用 command1 || exit 1, 如果 commands 1 执行失败，就执行 exit 1
# ls *.exe || exit 1

# 使用 -e 设置， set -o errexit
set -e

# 如果不存在 dll文件，不会终止，只要管道最后的命令执行成功
# 为了避免这种情况，可以使用 set -eo pipefail
ls *.dll | true

# 如果不存在 exe文件，错误终止
ls *.exe
echo "hi there"

# set +e 可以关闭该设置

setShellallowTrace.sh

#!/bin/bash

# set -o xtrace
set -x

echo "$PWD"

temp.sh

#!/bin/bash

# 脚本退出时，删除创建的临时文件
trap 'rm -f "$tmpFile"' EXIT

tmpFile=$(mktemp) || exit 1
echo "temporary file created successfully: $tmpFile"

逻辑卷管理 LVM

2022-08-13 Linux

基本的分区管理

Linux 用户通过路径 /xx 访问文件或者目录，其实通过 Linux 提供的虚拟目录进行访问，不同的文件存储设备的分区会挂载在虚拟目录下，或者是将远程的文件系统 (如 NFS) 目录挂载到虚拟目录下，进行本地访问。对本地文件存储设备首先需要进行分区，来容纳文件系统，来对文件进行访问和管理。

所以如果在安装硬盘后，如果需要进行使用前，需要进行：

1	创建分区 (fdsik 命令) --> 创建文件系统 (不同的文件系统命令，如 mkfs.ex4 ) --> 挂载到虚拟目录下的某个挂载点 (mount 命令)

创建分区分为主分区或扩展分区，因为扩展分区的个数由限制，所以通过扩展分区可以创建更多的逻辑分区，目前还有此限制吗？

通过 mount 命令进行挂载或查看当前挂载列表，也可以通过 cat /etc/fstab 查看启动挂载列表，如：

$ cat /etc/fstab
none /proc proc defaults 0 0
/dev/root / ext4 defaults 1 1
/dev/vg1/volume_1 /volume1 btrfs auto_reclaim_space,ssd,synoacl,relatime,nodev 0 0
/dev/vg2/volume_2 /volume2 btrfs auto_reclaim_space,ssd,synoacl,relatime,nodev 0 0

查看分区表以及某个分区的信息

$ sudo fdisk /dev/sda

Welcome to fdisk (util-linux 2.34).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.


Command (m for help): p
Disk /dev/sda: 111.81 GiB, 120034123776 bytes, 234441648 sectors
Disk model: Samsung SSD 850
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: gpt
Disk identifier: 3D330146-295F-494F-97D4-XXXXXXXXXXXX

Device       Start       End   Sectors   Size Type
/dev/sda1     2048   1050623   1048576   512M EFI System
/dev/sda2  1050624 234440703 233390080 111.3G Linux filesystem

Command (m for help): i
Partition number (1,2, default 2): 2

         Device: /dev/sda2
          Start: 1050624
            End: 234440703
        Sectors: 233390080
           Size: 111.3G
           Type: Linux filesystem
      Type-UUID: 0FC63DAF-8483-4772-8E79-XXXXXXXXXXXX
           UUID: 266FD3F0-CC07-4AE9-8C0A-XXXXXXXXXXXX

逻辑卷管理 LVM

1	物理卷 PV --> 卷组 VG --> 逻辑卷 LV

逻辑卷管理基本关系


|		逻辑卷1	    |     逻辑卷2				 |
--------------------------------------------------
|					卷组		                    |
--------------------------------------------------			
物理卷1	物理卷2	   物理卷3	  物理卷4		物理卷5
--------------------------------------------------
 分区1     分区2		 分区1     分区2		分区1		未使用
------------------------------------------------------------
      硬盘1				   硬盘2					硬盘3											
------------------------------------------------------------

文件系统是基于逻辑卷的，可以使用不同的存储设备对逻辑卷进行扩容

如我这台机器上就使用了 LVM 来管理磁盘，下面展示了卷组 vg2 是基于那些物理卷的 ( 在这里是 /dev/md3 )，以及在该卷组上创建的逻辑卷 (volume_2)

$ sudo vgdisplay -v  vg2
Password:
    Using volume group(s) on command line.
  --- Volume group ---
  VG Name               vg2
  System ID
  Format                lvm2
  Metadata Areas        1
  Metadata Sequence No  3
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                2
  Open LV               1
  Max PV                0
  Cur PV                1
  Act PV                1
  VG Size               3.63 TiB
  PE Size               4.00 MiB
  Total PE              951243
  Alloc PE / Size       951043 / 3.63 TiB
  Free  PE / Size       200 / 800.00 MiB
  VG UUID               


  --- Logical volume ---
  LV Path                /dev/vg2/volume_2
  LV Name                volume_2
  VG Name                vg2
  LV UUID                
  LV Write Access        read/write
  LV Creation host, time ,
  LV Status              available
  # open                 1
  LV Size                3.63 TiB
  Current LE             951040
  Segments               1
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     4096
  Block device           249:1

  --- Physical volumes ---
  PV Name               /dev/md3
  PV UUID               
  PV Status             allocatable
  Total PE / Free PE    951243 / 200

逻辑卷挂载信息：

1 2	$ mount \| grep vg2 /dev/mapper/vg2-volume_2 on /volume2 type btrfs (...)

1 2	$ cat /etc/fstab \| grep vg2 /dev/vg2/volume_2 /volume2 btrfs (...)

Hbase 术语与基本概念

2022-07-24 大数据

端口设置本地端口

Write-Ahead LOG HLog/WAL 预写式日志

MemStore 内存写入缓存区

BlockCache 读缓存

每台服务器维护一个 WAL 记录发生的变化, 这台服务器上的所有表和它们的列族共享这个 WAL

每个列族有一个 MemStore

BlockCache 保存从 HFile 里读入内存的频繁访问数据，避免硬盘读，每个列族有一个 BlockCache

Prev Next