Centos 配置ulimit，限制用户资源

绝地灬酷狼 2021-12-12 05:37 308阅读 0赞

### 文章目录 ###

*  1, OutOfMemoryError: unable to create new native thread
 *  2, 配置/etc/security/limits.conf文件

# 1, OutOfMemoryError: unable to create new native thread #

hdfs用户提交spark任务到yarn上报错：OutOfMemoryError: unable to create new native thread

java.lang.RuntimeException: java.lang.OutOfMemoryError: unable to create new native thread
            at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:218)
            at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:327)
            at org.apache.hadoop.hbase.client.ClientScanner.loadCache(ClientScanner.java:410)
            at org.apache.hadoop.hbase.client.ClientScanner.next(ClientScanner.java:371)
            at org.apache.hadoop.hbase.mapreduce.TableRecordReaderImpl.nextKeyValue(TableRecordReaderImpl.java:222)
            at org.apache.hadoop.hbase.mapreduce.TableRecordReader.nextKeyValue(TableRecordReader.java:147)
            at org.apache.hadoop.hbase.mapreduce.TableInputFormatBase$1.nextKeyValue(TableInputFormatBase.java:216)
            at org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:179)
            at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
            at org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1633)
            at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1157)
            at org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:1157)
            at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
            at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
            at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
            at org.apache.spark.scheduler.Task.run(Task.scala:89)
            at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242)
            at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
            at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
            at java.lang.Thread.run(Thread.java:748)
    Caused by: java.lang.OutOfMemoryError: unable to create new native thread
            at java.lang.Thread.start0(Native Method)
            at java.lang.Thread.start(Thread.java:717)
            at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:957)
            at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1367)
            at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.submit(ResultBoundedCompletionService.java:171)
            at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.addCallsForCurrentReplica(ScannerCallableWithReplicas.java:306)
            at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:171)
            at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:61)
            at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:210)
            ... 19 more

经排查，不是yarn分配的内存少，而是系统限制了内存使用

[hdfs@eadage ~]$ ulimit -a
    core file size          (blocks, -c) 0
    data seg size           (kbytes, -d) unlimited
    scheduling priority             (-e) 0
    file size               (blocks, -f) unlimited
    pending signals                 (-i) 256635
    max locked memory       (kbytes, -l) 64
    max memory size         (kbytes, -m) unlimited
    open files                      (-n) 1024
    pipe size            (512 bytes, -p) 8
    POSIX message queues     (bytes, -q) 819200
    real-time priority              (-r) 0
    stack size              (kbytes, -s) 10240
    cpu time               (seconds, -t) unlimited
    max user processes              (-u) 1024
    virtual memory          (kbytes, -v) unlimited
    file locks                      (-x) unlimited

# 2, 配置/etc/security/limits.conf文件 #

[root@cdh-node1 pam.d]# yum whatprovides /etc/security/limits.conf 
    pam-1.1.1-24.el6.x86_64 : An extensible library which provides authentication for applications
    Repo        : base
    Matched from:
    Filename    : /etc/security/limits.conf
    
    [root@cdh-node1 pam.d]#　tail /etc/security/limits.conf
    #内存限制
    hdfs            soft     memlock       10240
    hdfs            hard     memlock       unlimited
    hdfs            soft     stack          3072
    hdfs            hard     stack         nulimited
    #同时打开的文件数
    hdfs            soft     nofile         10240
    hdfs            hard     nofile        1000000
    ＃进程数
    hdfs            soft     nproc          2048
    hdfs            hard     nproc         nulimited

*  使配置立即生效，可修改/etc/pam.d/login

[vagrant@cdh-node1 pam.d]$ tail /etc/pam.d/login
    session    required     pam_loginuid.so
    session    optional     pam_console.so
    # pam_selinux.so open should only be followed by sessions to be executed in the user context
    session    required     pam_selinux.so open
    session    required     pam_namespace.so
    session    optional     pam_keyinit.so force revoke
    session    include      system-auth
    -session   optional     pam_ck_connector.so
    
    session required  pam_limits.so
    
    
    
    ##切换用户，查看是否生效
    [vagrant@cdh-node1 pam.d]$ sudo su hdfs
    [hdfs@cdh-node1 pam.d]$ ulimit -a
    core file size          (blocks, -c) 0
    data seg size           (kbytes, -d) unlimited
    scheduling priority             (-e) 0
    file size               (blocks, -f) unlimited
    pending signals                 (-i) 22824
    max locked memory       (kbytes, -l) 10240
    max memory size         (kbytes, -m) unlimited
    open files                      (-n) 10240
    pipe size            (512 bytes, -p) 8
    POSIX message queues     (bytes, -q) 819200
    real-time priority              (-r) 0
    stack size              (kbytes, -s) 3072
    cpu time               (seconds, -t) unlimited
    max user processes              (-u) 2048
    virtual memory          (kbytes, -v) unlimited
    file locks                      (-x) unlimited